From fe28f19f079e584c31ecbc66e253a98571eba4d3 Mon Sep 17 00:00:00 2001 From: Edward Shen Date: Tue, 11 May 2021 21:01:01 -0400 Subject: [PATCH] initial work into encryption at rest --- Cargo.lock | 195 +++++++++++++++++++++++++--------------------- Cargo.toml | 3 +- src/cache/disk.rs | 22 ++++-- src/cache/fs.rs | 38 +++++---- src/cache/mem.rs | 14 +++- src/cache/mod.rs | 55 ++++++++++++- src/config.rs | 20 +++-- src/main.rs | 25 +++--- src/ping.rs | 2 +- src/routes.rs | 6 +- src/stop.rs | 3 +- 11 files changed, 241 insertions(+), 142 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b184709..a32523f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,9 +96,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.0.0-beta.4" +version = "2.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0872f02a1871257ef09c5a269dce5dc5fea5ccf502adbf5d39f118913b61411c" +checksum = "26369215fcc3b0176018b3b68756a8bcc275bb000e6212e454944913a1f9bf87" dependencies = [ "actix-rt", "actix-service", @@ -227,9 +227,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -478,10 +478,13 @@ dependencies = [ ] [[package]] -name = "cpuid-bool" -version = "0.1.2" +name = "cpufeatures" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +checksum = "dec1028182c380cc45a2e2c5ec841134f2dfd0f8f5f0a5bcd68004f81b5efdf4" +dependencies = [ + "libc", +] [[package]] name = "crc" @@ -523,9 +526,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" dependencies = [ "autocfg", "cfg-if", @@ -626,9 +629,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "futures" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -641,9 +644,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -651,15 +654,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-executor" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" dependencies = [ "futures-core", "futures-task", @@ -668,16 +671,17 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-macro" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ + "autocfg", "proc-macro-hack", "proc-macro2", "quote", @@ -686,22 +690,23 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-util" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures-channel", "futures-core", "futures-io", @@ -739,9 +744,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" dependencies = [ "bytes", "fnv", @@ -811,9 +816,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ "bytes", "http", @@ -822,21 +827,21 @@ dependencies = [ [[package]] name = "httparse" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" +checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" -version = "0.3.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" [[package]] name = "hyper" -version = "0.14.5" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" +checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" dependencies = [ "bytes", "futures-channel", @@ -915,9 +920,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "js-sys" -version = "0.3.50" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" dependencies = [ "wasm-bindgen", ] @@ -936,9 +941,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lexical-core" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec", "bitflags", @@ -949,15 +954,15 @@ dependencies = [ [[package]] name = "lfu_cache" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33bfa6580d3aa7abe1f17d413dc9952d726eb588a0a8082821444cb89ffdabdf" +checksum = "29d2e56f95e5fda80586d85e2e98bb6dba8f71f4406161ce90698fa38ff16486" [[package]] name = "libc" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" [[package]] name = "libsodium-sys" @@ -972,9 +977,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19cb1effde5f834799ac5e5ef0e40d45027cd74f271b1de786ba8abb30e2164d" +checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d" dependencies = [ "cc", "pkg-config", @@ -1001,9 +1006,9 @@ checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" [[package]] name = "lock_api" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] @@ -1050,6 +1055,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "serde_repr", "simple_logger", "sodiumoxide", "sqlx", @@ -1074,9 +1080,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "mime" @@ -1380,18 +1386,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.4.5" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", @@ -1400,9 +1406,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.23" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "reqwest" @@ -1545,6 +1551,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98d0516900518c29efa217c298fa1f4e6c6ffc85ae29fd7f4ee48f176e1a9ed5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.0" @@ -1559,13 +1576,13 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.4" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" +checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" dependencies = [ "block-buffer", "cfg-if", - "cpuid-bool", + "cpufeatures", "digest", "opaque-debug", ] @@ -1578,13 +1595,13 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ "block-buffer", "cfg-if", - "cpuid-bool", + "cpufeatures", "digest", "opaque-debug", ] @@ -1613,9 +1630,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" [[package]] name = "smallvec" @@ -1831,9 +1848,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.69" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2", "quote", @@ -2034,9 +2051,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if", "pin-project-lite", @@ -2045,9 +2062,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" dependencies = [ "lazy_static", ] @@ -2096,9 +2113,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "unicode_categories" @@ -2114,9 +2131,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", @@ -2161,9 +2178,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" +checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if", "serde", @@ -2173,9 +2190,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" +checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static", @@ -2188,9 +2205,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" dependencies = [ "cfg-if", "js-sys", @@ -2200,9 +2217,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" +checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2210,9 +2227,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" +checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" dependencies = [ "proc-macro2", "quote", @@ -2223,15 +2240,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" +checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" [[package]] name = "web-sys" -version = "0.3.50" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 94711de..59f91a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/edward-shen/mangadex-home-rs" [profile.release] lto = true codegen-units = 1 -debug = 1 +# debug = 1 [dependencies] actix-web = { version = "4.0.0-beta.4", features = [ "rustls" ] } @@ -34,6 +34,7 @@ reqwest = { version = "0.11", default_features = false, features = [ "json", "st rustls = "0.19" serde = "1" serde_json = "1" +serde_repr = "0.1" simple_logger = "1" sodiumoxide = "0.2" sqlx = { version = "0.5", features = [ "runtime-actix-rustls", "sqlite", "time", "chrono", "macros" ] } diff --git a/src/cache/disk.rs b/src/cache/disk.rs index 7962bd9..8b3f0fd 100644 --- a/src/cache/disk.rs +++ b/src/cache/disk.rs @@ -182,9 +182,12 @@ impl Cache for DiskCache { tokio::spawn(async move { channel.send(DbMessage::Get(path_0)).await }); - super::fs::read_file(&path) - .await - .map(|res| res.map_err(Into::into)) + super::fs::read_file(&path).await.map(|res| { + let (inner, maybe_header, metadata) = res?; + CacheStream::new(inner, maybe_header) + .map(|stream| (stream, metadata)) + .map_err(|_| CacheError::DecryptionFailure) + }) } async fn put( @@ -204,7 +207,10 @@ impl Cache for DiskCache { super::fs::write_file(&path, key, image, metadata, db_callback, None) .await - .map_err(Into::into) + .map_err(CacheError::from) + .and_then(|(inner, maybe_header)| { + CacheStream::new(inner, maybe_header).map_err(|_| CacheError::DecryptionFailure) + }) } #[inline] @@ -241,12 +247,16 @@ impl Cache for DiskCache { let path_0 = Arc::clone(&path); let db_callback = |size: u32| async move { - let _ = channel.send(DbMessage::Put(path_0, size)).await; + // We don't care about the result of the send + std::mem::drop(channel.send(DbMessage::Put(path_0, size)).await); }; super::fs::write_file(&path, key, image, metadata, db_callback, Some(on_complete)) .await - .map_err(Into::into) + .map_err(CacheError::from) + .and_then(|(inner, maybe_header)| { + CacheStream::new(inner, maybe_header).map_err(|_| CacheError::DecryptionFailure) + }) } #[inline] diff --git a/src/cache/fs.rs b/src/cache/fs.rs index eb891e6..38259f0 100644 --- a/src/cache/fs.rs +++ b/src/cache/fs.rs @@ -19,7 +19,8 @@ use bytes::{Buf, Bytes, BytesMut}; use futures::{Future, Stream, StreamExt}; use log::debug; use once_cell::sync::Lazy; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use sodiumoxide::crypto::secretstream::Header; use std::collections::HashMap; use std::error::Error; use std::fmt::Display; @@ -36,7 +37,13 @@ use tokio::sync::RwLock; use tokio_stream::wrappers::WatchStream; use tokio_util::codec::{BytesCodec, FramedRead}; -use super::{BoxedImageStream, CacheKey, CacheStream, CacheStreamItem, ImageMetadata}; +use super::{BoxedImageStream, CacheKey, CacheStreamItem, ImageMetadata, InnerStream}; + +#[derive(Serialize, Deserialize)] +pub enum OnDiskMetadata { + Encrypted(Header, ImageMetadata), + Plaintext(ImageMetadata), +} /// Keeps track of files that are currently being written to. /// @@ -60,9 +67,9 @@ static WRITING_STATUS: Lazy>>> = /// Attempts to lookup the file on disk, returning a byte stream if it exists. /// Note that this could return two types of streams, depending on if the file /// is in progress of being written to. -pub async fn read_file( +pub(super) async fn read_file( path: &Path, -) -> Option> { +) -> Option, ImageMetadata), std::io::Error>> { let std_file = std::fs::File::open(path).ok()?; let file = File::from_std(std_file.try_clone().ok()?); @@ -75,29 +82,29 @@ pub async fn read_file( // not possible to move this into a map_or_else without cloning `file`. #[allow(clippy::option_if_let_else)] let stream = if let Some(status) = WRITING_STATUS.read().await.get(path).map(Clone::clone) { - CacheStream::Concurrent(ConcurrentFsStream::from_file( + InnerStream::Concurrent(ConcurrentFsStream::from_file( file, WatchStream::new(status), )) } else { - CacheStream::Completed(FramedRead::new(BufReader::new(file), BytesCodec::new())) + InnerStream::Completed(FramedRead::new(BufReader::new(file), BytesCodec::new())) }; - Some(Ok((stream, metadata))) + Some(Ok((stream, None, metadata))) } /// Writes the metadata and input stream (in that order) to a file, returning a /// stream that reads from that file. Accepts a db callback function that is /// provided the number of bytes written, and an optional on-complete callback /// that is called with a completed cache entry. -pub async fn write_file( +pub(super) async fn write_file( path: &Path, cache_key: CacheKey, mut byte_stream: BoxedImageStream, metadata: ImageMetadata, db_callback: DbCallback, on_complete: Option>, -) -> Result +) -> Result<(InnerStream, Option
), std::io::Error> where Fut: 'static + Send + Sync + Future, DbCallback: 'static + Send + Sync + FnOnce(u32) -> Fut, @@ -150,8 +157,10 @@ where if errored { // It's ok if the deleting the file fails, since we truncate on - // create anyways, but it should be best effort - let _ = remove_file(&path_buf).await; + // create anyways, but it should be best effort. + // + // We don't care about the result of the call. + std::mem::drop(remove_file(&path_buf).await); } else { file.flush().await?; file.sync_all().await?; // we need metadata @@ -191,8 +200,11 @@ where Ok::<_, std::io::Error>(()) }); - Ok(CacheStream::Concurrent( - ConcurrentFsStream::new(path, metadata_size, WatchStream::new(rx)).await?, + Ok(( + InnerStream::Concurrent( + ConcurrentFsStream::new(path, metadata_size, WatchStream::new(rx)).await?, + ), + None, )) } diff --git a/src/cache/mem.rs b/src/cache/mem.rs index 6bcd428..7276ce2 100644 --- a/src/cache/mem.rs +++ b/src/cache/mem.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use crate::cache::DiskCache; -use super::{BoxedImageStream, Cache, CacheKey, CacheStream, ImageMetadata, MemStream}; +use super::{ + BoxedImageStream, Cache, CacheKey, CacheStream, ImageMetadata, InnerStream, MemStream, +}; use async_trait::async_trait; use bytes::Bytes; use futures::FutureExt; @@ -15,9 +17,12 @@ use tokio::sync::Mutex; type CacheValue = (Bytes, ImageMetadata, usize); +/// Use LRU as the eviction strategy pub type Lru = LruCache; +/// Use LFU as the eviction strategy pub type Lfu = LfuCache; +/// Adapter trait for memory cache backends pub trait InternalMemoryCache: Sync + Send { fn unbounded() -> Self; fn get(&mut self, key: &CacheKey) -> Option<&CacheValue>; @@ -25,7 +30,8 @@ pub trait InternalMemoryCache: Sync + Send { fn pop(&mut self) -> Option<(CacheKey, CacheValue)>; } -/// Memory accelerated disk cache. Uses an LRU in memory to speed up reads. +/// Memory accelerated disk cache. Uses the internal cache implementation in +/// memory to speed up reads. pub struct MemoryCache { inner: Arc>, cur_mem_size: AtomicU64, @@ -78,9 +84,9 @@ impl Cache for MemoryCache Option> { match self.mem_cache.lock().now_or_never() { Some(mut mem_cache) => match mem_cache.get(key).map(|(bytes, metadata, _)| { - Ok((CacheStream::Memory(MemStream(bytes.clone())), *metadata)) + Ok((InnerStream::Memory(MemStream(bytes.clone())), *metadata)) }) { - Some(v) => Some(v), + Some(v) => Some(v.map(|(inner, metadata)| todo!())), None => self.inner.get(key).await, }, None => self.inner.get(key).await, diff --git a/src/cache/mod.rs b/src/cache/mod.rs index ce5a6d9..0152516 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -10,7 +10,10 @@ use bytes::{Bytes, BytesMut}; use chrono::{DateTime, FixedOffset}; use fs::ConcurrentFsStream; use futures::{Stream, StreamExt}; +use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; +use serde_repr::{Deserialize_repr, Serialize_repr}; +use sodiumoxide::crypto::secretstream::{gen_key, Header, Key, Pull, Stream as SecretStream}; use thiserror::Error; use tokio::fs::File; use tokio::io::BufReader; @@ -21,6 +24,8 @@ pub use disk::DiskCache; pub use fs::UpstreamError; pub use mem::MemoryCache; +static ENCRYPTION_KEY: Lazy = Lazy::new(gen_key); + mod disk; mod fs; pub mod mem; @@ -63,7 +68,8 @@ pub struct ImageMetadata { } // Confirmed by Ply to be these types: https://link.eddie.sh/ZXfk0 -#[derive(Copy, Clone, Serialize, Deserialize)] +#[derive(Copy, Clone, Serialize_repr, Deserialize_repr)] +#[repr(u8)] pub enum ImageContentType { Png = 0, Jpeg, @@ -152,6 +158,8 @@ pub enum CacheError { Reqwest(#[from] reqwest::Error), #[error(transparent)] Upstream(#[from] UpstreamError), + #[error("An error occurred while reading the decryption header")] + DecryptionFailure, } #[async_trait] @@ -198,13 +206,52 @@ pub trait Cache: Send + Sync { async fn pop_memory(&self) -> Option<(CacheKey, Bytes, ImageMetadata, usize)>; } -pub enum CacheStream { +pub struct CacheStream { + inner: InnerStream, + decrypt: Option>, +} + +impl CacheStream { + pub(self) fn new(inner: InnerStream, header: Option
) -> Result { + Ok(Self { + inner, + decrypt: header + .map(|header| SecretStream::init_pull(&header, &ENCRYPTION_KEY)) + .transpose()?, + }) + } +} + +impl Stream for CacheStream { + type Item = CacheStreamItem; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_next_unpin(cx).map(|data| { + // False positive (`data`): https://link.eddie.sh/r1fXX + #[allow(clippy::option_if_let_else)] + if let Some(keystream) = self.decrypt.as_mut() { + data.map(|bytes_res| { + bytes_res.and_then(|bytes| { + keystream + .pull(&bytes, None) + .map(|(data, _tag)| Bytes::from(data)) + .map_err(|_| UpstreamError) + }) + }) + } else { + data + } + }) + } +} + +pub(self) enum InnerStream { Concurrent(ConcurrentFsStream), Memory(MemStream), Completed(FramedRead, BytesCodec>), } -impl From for CacheStream { +impl From for InnerStream { fn from(image: CachedImage) -> Self { Self::Memory(MemStream(image.0)) } @@ -212,7 +259,7 @@ impl From for CacheStream { type CacheStreamItem = Result; -impl Stream for CacheStream { +impl Stream for InnerStream { type Item = CacheStreamItem; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { diff --git a/src/config.rs b/src/config.rs index ae176b2..87df0e2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,9 +1,8 @@ -use std::{fmt::Display, path::PathBuf}; -use std::{fmt::Formatter, sync::atomic::AtomicBool}; -use std::{ - num::{NonZeroU16, NonZeroU64}, - str::FromStr, -}; +use std::fmt::{Display, Formatter}; +use std::num::{NonZeroU16, NonZeroU64}; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::atomic::AtomicBool; use clap::{crate_authors, crate_description, crate_version, Clap}; use url::Url; @@ -22,8 +21,8 @@ pub struct CliArgs { pub port: NonZeroU16, /// How large, in bytes, the in-memory cache should be. Note that this does /// not include runtime memory usage. - #[clap(long, env = "MEM_CACHE_QUOTA_BYTES")] - pub memory_quota: NonZeroU64, + #[clap(long, env = "MEM_CACHE_QUOTA_BYTES", conflicts_with = "low-memory")] + pub memory_quota: Option, /// How large, in bytes, the on-disk cache should be. Note that actual /// values may be larger for metadata information. #[clap(long, env = "DISK_CACHE_QUOTA_BYTES")] @@ -63,6 +62,11 @@ pub struct CliArgs { pub unstable_options: Vec, #[clap(long)] pub override_upstream: Option, + /// Enables ephemeral disk encryption. Items written to disk are first + /// encrypted with a key generated at runtime. There are implications to + /// performance, privacy, and usability with this flag enabled. + #[clap(short, long)] + pub ephemeral_disk_encryption: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/src/main.rs b/src/main.rs index ae5f9ad..45ae2ea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,16 +2,15 @@ // We're end users, so these is ok #![allow(clippy::module_name_repetitions)] -use std::num::ParseIntError; +use std::env::{self, VarError}; +use std::error::Error; +use std::fmt::Display; +use std::hint::unreachable_unchecked; +use std::num::{NonZeroU64, ParseIntError}; use std::process; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use std::{ - env::{self, VarError}, - fmt::Display, -}; -use std::{error::Error, hint::unreachable_unchecked}; use actix_web::rt::{spawn, time, System}; use actix_web::web::{self, Data}; @@ -55,12 +54,16 @@ enum ServerError { #[actix_web::main] async fn main() -> Result<(), Box> { + sodiumoxide::init().expect("Failed to initialize crypto"); // It's ok to fail early here, it would imply we have a invalid config. dotenv::dotenv().ok(); let cli_args = CliArgs::parse(); let port = cli_args.port; - let memory_max_size = cli_args.memory_quota.get(); + let memory_max_size = cli_args + .memory_quota + .map(NonZeroU64::get) + .unwrap_or_default(); let disk_quota = cli_args.disk_quota; let cache_path = cli_args.cache_path.clone(); let low_mem_mode = cli_args.low_memory; @@ -137,12 +140,10 @@ async fn main() -> Result<(), Box> { let cache: Arc> = if low_mem_mode { DiskCache::new(disk_quota, cache_path.clone()).await + } else if use_lfu { + MemoryCache::::new(disk_quota, cache_path.clone(), memory_max_size).await } else { - if use_lfu { - MemoryCache::::new(disk_quota, cache_path.clone(), memory_max_size).await - } else { - MemoryCache::::new(disk_quota, cache_path.clone(), memory_max_size).await - } + MemoryCache::::new(disk_quota, cache_path.clone(), memory_max_size).await }; let cache_0 = Arc::clone(&cache); diff --git a/src/ping.rs b/src/ping.rs index ecd647a..553411e 100644 --- a/src/ping.rs +++ b/src/ping.rs @@ -42,7 +42,7 @@ impl<'a> Request<'a> { .expect("to parse the build version"), tls_created_at: TLS_PREVIOUSLY_CREATED .get() - .map(|v| v.load().as_ref().to_owned()), + .map(|v| v.load().as_ref().clone()), } } } diff --git a/src/routes.rs b/src/routes.rs index 02a99e9..0bd41b9 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -13,7 +13,7 @@ use chrono::{DateTime, Utc}; use futures::{Stream, TryStreamExt}; use log::{debug, error, info, warn}; use once_cell::sync::Lazy; -use reqwest::Client; +use reqwest::{Client, StatusCode}; use serde::Deserialize; use sodiumoxide::crypto::box_::{open_precomputed, Nonce, PrecomputedKey, NONCEBYTES}; use thiserror::Error; @@ -238,7 +238,7 @@ async fn fetch_image( .map(|v| String::from_utf8_lossy(v.as_ref()).contains("image/")) .unwrap_or_default(); - if resp.status() != 200 || !is_image { + if resp.status() != StatusCode::OK || !is_image { warn!( "Got non-OK or non-image response code from upstream, proxying and not caching result.", ); @@ -283,7 +283,7 @@ async fn fetch_image( debug!("Done putting into cache"); - return construct_response(stream, &metadata); + construct_response(stream, &metadata) } Err(e) => { error!("Failed to fetch image from server: {}", e); diff --git a/src/stop.rs b/src/stop.rs index e61a176..723f02e 100644 --- a/src/stop.rs +++ b/src/stop.rs @@ -1,4 +1,5 @@ use log::{info, warn}; +use reqwest::StatusCode; use serde::Serialize; const CONTROL_CENTER_STOP_URL: &str = "https://api.mangadex.network/ping"; @@ -18,7 +19,7 @@ pub async fn send_stop(secret: &str) { .await { Ok(resp) => { - if resp.status() == 200 { + if resp.status() == StatusCode::OK { info!("Successfully sent stop message to control center."); } else { warn!("Got weird response from server: {:?}", resp.headers());