initial work into encryption at rest

This commit is contained in:
Edward Shen 2021-05-11 21:01:01 -04:00
parent eb605b9c27
commit fe28f19f07
Signed by: edward
GPG key ID: 19182661E818369F
11 changed files with 241 additions and 142 deletions

195
Cargo.lock generated
View file

@ -96,9 +96,9 @@ dependencies = [
[[package]]
name = "actix-server"
version = "2.0.0-beta.4"
version = "2.0.0-beta.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0872f02a1871257ef09c5a269dce5dc5fea5ccf502adbf5d39f118913b61411c"
checksum = "26369215fcc3b0176018b3b68756a8bcc275bb000e6212e454944913a1f9bf87"
dependencies = [
"actix-rt",
"actix-service",
@ -227,9 +227,9 @@ dependencies = [
[[package]]
name = "aho-corasick"
version = "0.7.15"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [
"memchr",
]
@ -478,10 +478,13 @@ dependencies = [
]
[[package]]
name = "cpuid-bool"
version = "0.1.2"
name = "cpufeatures"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
checksum = "dec1028182c380cc45a2e2c5ec841134f2dfd0f8f5f0a5bcd68004f81b5efdf4"
dependencies = [
"libc",
]
[[package]]
name = "crc"
@ -523,9 +526,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278"
dependencies = [
"autocfg",
"cfg-if",
@ -626,9 +629,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
[[package]]
name = "futures"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253"
checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27"
dependencies = [
"futures-channel",
"futures-core",
@ -641,9 +644,9 @@ dependencies = [
[[package]]
name = "futures-channel"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25"
checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
dependencies = [
"futures-core",
"futures-sink",
@ -651,15 +654,15 @@ dependencies = [
[[package]]
name = "futures-core"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815"
checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
[[package]]
name = "futures-executor"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d"
checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79"
dependencies = [
"futures-core",
"futures-task",
@ -668,16 +671,17 @@ dependencies = [
[[package]]
name = "futures-io"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04"
checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
[[package]]
name = "futures-macro"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b"
checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121"
dependencies = [
"autocfg",
"proc-macro-hack",
"proc-macro2",
"quote",
@ -686,22 +690,23 @@ dependencies = [
[[package]]
name = "futures-sink"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23"
checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
[[package]]
name = "futures-task"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc"
checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
[[package]]
name = "futures-util"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025"
checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
dependencies = [
"autocfg",
"futures-channel",
"futures-core",
"futures-io",
@ -739,9 +744,9 @@ dependencies = [
[[package]]
name = "h2"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00"
checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726"
dependencies = [
"bytes",
"fnv",
@ -811,9 +816,9 @@ dependencies = [
[[package]]
name = "http-body"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfb77c123b4e2f72a2069aeae0b4b4949cc7e966df277813fc16347e7549737"
checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
dependencies = [
"bytes",
"http",
@ -822,21 +827,21 @@ dependencies = [
[[package]]
name = "httparse"
version = "1.4.0"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437"
checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"
[[package]]
name = "httpdate"
version = "0.3.2"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47"
checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9"
[[package]]
name = "hyper"
version = "0.14.5"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1"
checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54"
dependencies = [
"bytes",
"futures-channel",
@ -915,9 +920,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
[[package]]
name = "js-sys"
version = "0.3.50"
version = "0.3.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c"
checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
dependencies = [
"wasm-bindgen",
]
@ -936,9 +941,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lexical-core"
version = "0.7.5"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374"
checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
dependencies = [
"arrayvec",
"bitflags",
@ -949,15 +954,15 @@ dependencies = [
[[package]]
name = "lfu_cache"
version = "1.2.0"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33bfa6580d3aa7abe1f17d413dc9952d726eb588a0a8082821444cb89ffdabdf"
checksum = "29d2e56f95e5fda80586d85e2e98bb6dba8f71f4406161ce90698fa38ff16486"
[[package]]
name = "libc"
version = "0.2.93"
version = "0.2.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
[[package]]
name = "libsodium-sys"
@ -972,9 +977,9 @@ dependencies = [
[[package]]
name = "libsqlite3-sys"
version = "0.22.1"
version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19cb1effde5f834799ac5e5ef0e40d45027cd74f271b1de786ba8abb30e2164d"
checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d"
dependencies = [
"cc",
"pkg-config",
@ -1001,9 +1006,9 @@ checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4"
[[package]]
name = "lock_api"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176"
checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
dependencies = [
"scopeguard",
]
@ -1050,6 +1055,7 @@ dependencies = [
"rustls",
"serde",
"serde_json",
"serde_repr",
"simple_logger",
"sodiumoxide",
"sqlx",
@ -1074,9 +1080,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "memchr"
version = "2.3.4"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
[[package]]
name = "mime"
@ -1380,18 +1386,18 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.2.6"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041"
checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.4.5"
version = "1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
dependencies = [
"aho-corasick",
"memchr",
@ -1400,9 +1406,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.6.23"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "reqwest"
@ -1545,6 +1551,17 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_repr"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98d0516900518c29efa217c298fa1f4e6c6ffc85ae29fd7f4ee48f176e1a9ed5"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.0"
@ -1559,13 +1576,13 @@ dependencies = [
[[package]]
name = "sha-1"
version = "0.9.4"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f"
checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16"
dependencies = [
"block-buffer",
"cfg-if",
"cpuid-bool",
"cpufeatures",
"digest",
"opaque-debug",
]
@ -1578,13 +1595,13 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d"
[[package]]
name = "sha2"
version = "0.9.3"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de"
checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
dependencies = [
"block-buffer",
"cfg-if",
"cpuid-bool",
"cpufeatures",
"digest",
"opaque-debug",
]
@ -1613,9 +1630,9 @@ dependencies = [
[[package]]
name = "slab"
version = "0.4.2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
[[package]]
name = "smallvec"
@ -1831,9 +1848,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
version = "1.0.69"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb"
checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
dependencies = [
"proc-macro2",
"quote",
@ -2034,9 +2051,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
version = "0.1.25"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f"
checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d"
dependencies = [
"cfg-if",
"pin-project-lite",
@ -2045,9 +2062,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.17"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f"
checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052"
dependencies = [
"lazy_static",
]
@ -2096,9 +2113,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "unicode_categories"
@ -2114,9 +2131,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "url"
version = "2.2.1"
version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b"
checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
dependencies = [
"form_urlencoded",
"idna",
@ -2161,9 +2178,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.73"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9"
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
dependencies = [
"cfg-if",
"serde",
@ -2173,9 +2190,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.73"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae"
checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
dependencies = [
"bumpalo",
"lazy_static",
@ -2188,9 +2205,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.23"
version = "0.4.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea"
checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
dependencies = [
"cfg-if",
"js-sys",
@ -2200,9 +2217,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.73"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f"
checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -2210,9 +2227,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.73"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c"
checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
dependencies = [
"proc-macro2",
"quote",
@ -2223,15 +2240,15 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.73"
version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489"
checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
[[package]]
name = "web-sys"
version = "0.3.50"
version = "0.3.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be"
checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582"
dependencies = [
"js-sys",
"wasm-bindgen",

View file

@ -11,7 +11,7 @@ repository = "https://github.com/edward-shen/mangadex-home-rs"
[profile.release]
lto = true
codegen-units = 1
debug = 1
# debug = 1
[dependencies]
actix-web = { version = "4.0.0-beta.4", features = [ "rustls" ] }
@ -34,6 +34,7 @@ reqwest = { version = "0.11", default_features = false, features = [ "json", "st
rustls = "0.19"
serde = "1"
serde_json = "1"
serde_repr = "0.1"
simple_logger = "1"
sodiumoxide = "0.2"
sqlx = { version = "0.5", features = [ "runtime-actix-rustls", "sqlite", "time", "chrono", "macros" ] }

22
src/cache/disk.rs vendored
View file

@ -182,9 +182,12 @@ impl Cache for DiskCache {
tokio::spawn(async move { channel.send(DbMessage::Get(path_0)).await });
super::fs::read_file(&path)
.await
.map(|res| res.map_err(Into::into))
super::fs::read_file(&path).await.map(|res| {
let (inner, maybe_header, metadata) = res?;
CacheStream::new(inner, maybe_header)
.map(|stream| (stream, metadata))
.map_err(|_| CacheError::DecryptionFailure)
})
}
async fn put(
@ -204,7 +207,10 @@ impl Cache for DiskCache {
super::fs::write_file(&path, key, image, metadata, db_callback, None)
.await
.map_err(Into::into)
.map_err(CacheError::from)
.and_then(|(inner, maybe_header)| {
CacheStream::new(inner, maybe_header).map_err(|_| CacheError::DecryptionFailure)
})
}
#[inline]
@ -241,12 +247,16 @@ impl Cache for DiskCache {
let path_0 = Arc::clone(&path);
let db_callback = |size: u32| async move {
let _ = channel.send(DbMessage::Put(path_0, size)).await;
// We don't care about the result of the send
std::mem::drop(channel.send(DbMessage::Put(path_0, size)).await);
};
super::fs::write_file(&path, key, image, metadata, db_callback, Some(on_complete))
.await
.map_err(Into::into)
.map_err(CacheError::from)
.and_then(|(inner, maybe_header)| {
CacheStream::new(inner, maybe_header).map_err(|_| CacheError::DecryptionFailure)
})
}
#[inline]

38
src/cache/fs.rs vendored
View file

@ -19,7 +19,8 @@ use bytes::{Buf, Bytes, BytesMut};
use futures::{Future, Stream, StreamExt};
use log::debug;
use once_cell::sync::Lazy;
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use sodiumoxide::crypto::secretstream::Header;
use std::collections::HashMap;
use std::error::Error;
use std::fmt::Display;
@ -36,7 +37,13 @@ use tokio::sync::RwLock;
use tokio_stream::wrappers::WatchStream;
use tokio_util::codec::{BytesCodec, FramedRead};
use super::{BoxedImageStream, CacheKey, CacheStream, CacheStreamItem, ImageMetadata};
use super::{BoxedImageStream, CacheKey, CacheStreamItem, ImageMetadata, InnerStream};
#[derive(Serialize, Deserialize)]
pub enum OnDiskMetadata {
Encrypted(Header, ImageMetadata),
Plaintext(ImageMetadata),
}
/// Keeps track of files that are currently being written to.
///
@ -60,9 +67,9 @@ static WRITING_STATUS: Lazy<RwLock<HashMap<PathBuf, Receiver<WritingStatus>>>> =
/// Attempts to lookup the file on disk, returning a byte stream if it exists.
/// Note that this could return two types of streams, depending on if the file
/// is in progress of being written to.
pub async fn read_file(
pub(super) async fn read_file(
path: &Path,
) -> Option<Result<(CacheStream, ImageMetadata), std::io::Error>> {
) -> Option<Result<(InnerStream, Option<Header>, ImageMetadata), std::io::Error>> {
let std_file = std::fs::File::open(path).ok()?;
let file = File::from_std(std_file.try_clone().ok()?);
@ -75,29 +82,29 @@ pub async fn read_file(
// not possible to move this into a map_or_else without cloning `file`.
#[allow(clippy::option_if_let_else)]
let stream = if let Some(status) = WRITING_STATUS.read().await.get(path).map(Clone::clone) {
CacheStream::Concurrent(ConcurrentFsStream::from_file(
InnerStream::Concurrent(ConcurrentFsStream::from_file(
file,
WatchStream::new(status),
))
} else {
CacheStream::Completed(FramedRead::new(BufReader::new(file), BytesCodec::new()))
InnerStream::Completed(FramedRead::new(BufReader::new(file), BytesCodec::new()))
};
Some(Ok((stream, metadata)))
Some(Ok((stream, None, metadata)))
}
/// Writes the metadata and input stream (in that order) to a file, returning a
/// stream that reads from that file. Accepts a db callback function that is
/// provided the number of bytes written, and an optional on-complete callback
/// that is called with a completed cache entry.
pub async fn write_file<Fut, DbCallback>(
pub(super) async fn write_file<Fut, DbCallback>(
path: &Path,
cache_key: CacheKey,
mut byte_stream: BoxedImageStream,
metadata: ImageMetadata,
db_callback: DbCallback,
on_complete: Option<Sender<(CacheKey, Bytes, ImageMetadata, usize)>>,
) -> Result<CacheStream, std::io::Error>
) -> Result<(InnerStream, Option<Header>), std::io::Error>
where
Fut: 'static + Send + Sync + Future<Output = ()>,
DbCallback: 'static + Send + Sync + FnOnce(u32) -> Fut,
@ -150,8 +157,10 @@ where
if errored {
// It's ok if the deleting the file fails, since we truncate on
// create anyways, but it should be best effort
let _ = remove_file(&path_buf).await;
// create anyways, but it should be best effort.
//
// We don't care about the result of the call.
std::mem::drop(remove_file(&path_buf).await);
} else {
file.flush().await?;
file.sync_all().await?; // we need metadata
@ -191,8 +200,11 @@ where
Ok::<_, std::io::Error>(())
});
Ok(CacheStream::Concurrent(
ConcurrentFsStream::new(path, metadata_size, WatchStream::new(rx)).await?,
Ok((
InnerStream::Concurrent(
ConcurrentFsStream::new(path, metadata_size, WatchStream::new(rx)).await?,
),
None,
))
}

14
src/cache/mem.rs vendored
View file

@ -4,7 +4,9 @@ use std::sync::Arc;
use crate::cache::DiskCache;
use super::{BoxedImageStream, Cache, CacheKey, CacheStream, ImageMetadata, MemStream};
use super::{
BoxedImageStream, Cache, CacheKey, CacheStream, ImageMetadata, InnerStream, MemStream,
};
use async_trait::async_trait;
use bytes::Bytes;
use futures::FutureExt;
@ -15,9 +17,12 @@ use tokio::sync::Mutex;
type CacheValue = (Bytes, ImageMetadata, usize);
/// Use LRU as the eviction strategy
pub type Lru = LruCache<CacheKey, CacheValue>;
/// Use LFU as the eviction strategy
pub type Lfu = LfuCache<CacheKey, CacheValue>;
/// Adapter trait for memory cache backends
pub trait InternalMemoryCache: Sync + Send {
fn unbounded() -> Self;
fn get(&mut self, key: &CacheKey) -> Option<&CacheValue>;
@ -25,7 +30,8 @@ pub trait InternalMemoryCache: Sync + Send {
fn pop(&mut self) -> Option<(CacheKey, CacheValue)>;
}
/// Memory accelerated disk cache. Uses an LRU in memory to speed up reads.
/// Memory accelerated disk cache. Uses the internal cache implementation in
/// memory to speed up reads.
pub struct MemoryCache<InternalCacheImpl> {
inner: Arc<Box<dyn Cache>>,
cur_mem_size: AtomicU64,
@ -78,9 +84,9 @@ impl<InternalCacheImpl: InternalMemoryCache> Cache for MemoryCache<InternalCache
) -> Option<Result<(CacheStream, ImageMetadata), super::CacheError>> {
match self.mem_cache.lock().now_or_never() {
Some(mut mem_cache) => match mem_cache.get(key).map(|(bytes, metadata, _)| {
Ok((CacheStream::Memory(MemStream(bytes.clone())), *metadata))
Ok((InnerStream::Memory(MemStream(bytes.clone())), *metadata))
}) {
Some(v) => Some(v),
Some(v) => Some(v.map(|(inner, metadata)| todo!())),
None => self.inner.get(key).await,
},
None => self.inner.get(key).await,

55
src/cache/mod.rs vendored
View file

@ -10,7 +10,10 @@ use bytes::{Bytes, BytesMut};
use chrono::{DateTime, FixedOffset};
use fs::ConcurrentFsStream;
use futures::{Stream, StreamExt};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use serde_repr::{Deserialize_repr, Serialize_repr};
use sodiumoxide::crypto::secretstream::{gen_key, Header, Key, Pull, Stream as SecretStream};
use thiserror::Error;
use tokio::fs::File;
use tokio::io::BufReader;
@ -21,6 +24,8 @@ pub use disk::DiskCache;
pub use fs::UpstreamError;
pub use mem::MemoryCache;
static ENCRYPTION_KEY: Lazy<Key> = Lazy::new(gen_key);
mod disk;
mod fs;
pub mod mem;
@ -63,7 +68,8 @@ pub struct ImageMetadata {
}
// Confirmed by Ply to be these types: https://link.eddie.sh/ZXfk0
#[derive(Copy, Clone, Serialize, Deserialize)]
#[derive(Copy, Clone, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum ImageContentType {
Png = 0,
Jpeg,
@ -152,6 +158,8 @@ pub enum CacheError {
Reqwest(#[from] reqwest::Error),
#[error(transparent)]
Upstream(#[from] UpstreamError),
#[error("An error occurred while reading the decryption header")]
DecryptionFailure,
}
#[async_trait]
@ -198,13 +206,52 @@ pub trait Cache: Send + Sync {
async fn pop_memory(&self) -> Option<(CacheKey, Bytes, ImageMetadata, usize)>;
}
pub enum CacheStream {
pub struct CacheStream {
inner: InnerStream,
decrypt: Option<SecretStream<Pull>>,
}
impl CacheStream {
pub(self) fn new(inner: InnerStream, header: Option<Header>) -> Result<Self, ()> {
Ok(Self {
inner,
decrypt: header
.map(|header| SecretStream::init_pull(&header, &ENCRYPTION_KEY))
.transpose()?,
})
}
}
impl Stream for CacheStream {
type Item = CacheStreamItem;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.inner.poll_next_unpin(cx).map(|data| {
// False positive (`data`): https://link.eddie.sh/r1fXX
#[allow(clippy::option_if_let_else)]
if let Some(keystream) = self.decrypt.as_mut() {
data.map(|bytes_res| {
bytes_res.and_then(|bytes| {
keystream
.pull(&bytes, None)
.map(|(data, _tag)| Bytes::from(data))
.map_err(|_| UpstreamError)
})
})
} else {
data
}
})
}
}
pub(self) enum InnerStream {
Concurrent(ConcurrentFsStream),
Memory(MemStream),
Completed(FramedRead<BufReader<File>, BytesCodec>),
}
impl From<CachedImage> for CacheStream {
impl From<CachedImage> for InnerStream {
fn from(image: CachedImage) -> Self {
Self::Memory(MemStream(image.0))
}
@ -212,7 +259,7 @@ impl From<CachedImage> for CacheStream {
type CacheStreamItem = Result<Bytes, UpstreamError>;
impl Stream for CacheStream {
impl Stream for InnerStream {
type Item = CacheStreamItem;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {

View file

@ -1,9 +1,8 @@
use std::{fmt::Display, path::PathBuf};
use std::{fmt::Formatter, sync::atomic::AtomicBool};
use std::{
num::{NonZeroU16, NonZeroU64},
str::FromStr,
};
use std::fmt::{Display, Formatter};
use std::num::{NonZeroU16, NonZeroU64};
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::atomic::AtomicBool;
use clap::{crate_authors, crate_description, crate_version, Clap};
use url::Url;
@ -22,8 +21,8 @@ pub struct CliArgs {
pub port: NonZeroU16,
/// How large, in bytes, the in-memory cache should be. Note that this does
/// not include runtime memory usage.
#[clap(long, env = "MEM_CACHE_QUOTA_BYTES")]
pub memory_quota: NonZeroU64,
#[clap(long, env = "MEM_CACHE_QUOTA_BYTES", conflicts_with = "low-memory")]
pub memory_quota: Option<NonZeroU64>,
/// How large, in bytes, the on-disk cache should be. Note that actual
/// values may be larger for metadata information.
#[clap(long, env = "DISK_CACHE_QUOTA_BYTES")]
@ -63,6 +62,11 @@ pub struct CliArgs {
pub unstable_options: Vec<UnstableOptions>,
#[clap(long)]
pub override_upstream: Option<Url>,
/// Enables ephemeral disk encryption. Items written to disk are first
/// encrypted with a key generated at runtime. There are implications to
/// performance, privacy, and usability with this flag enabled.
#[clap(short, long)]
pub ephemeral_disk_encryption: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]

View file

@ -2,16 +2,15 @@
// We're end users, so these is ok
#![allow(clippy::module_name_repetitions)]
use std::num::ParseIntError;
use std::env::{self, VarError};
use std::error::Error;
use std::fmt::Display;
use std::hint::unreachable_unchecked;
use std::num::{NonZeroU64, ParseIntError};
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use std::{
env::{self, VarError},
fmt::Display,
};
use std::{error::Error, hint::unreachable_unchecked};
use actix_web::rt::{spawn, time, System};
use actix_web::web::{self, Data};
@ -55,12 +54,16 @@ enum ServerError {
#[actix_web::main]
async fn main() -> Result<(), Box<dyn Error>> {
sodiumoxide::init().expect("Failed to initialize crypto");
// It's ok to fail early here, it would imply we have a invalid config.
dotenv::dotenv().ok();
let cli_args = CliArgs::parse();
let port = cli_args.port;
let memory_max_size = cli_args.memory_quota.get();
let memory_max_size = cli_args
.memory_quota
.map(NonZeroU64::get)
.unwrap_or_default();
let disk_quota = cli_args.disk_quota;
let cache_path = cli_args.cache_path.clone();
let low_mem_mode = cli_args.low_memory;
@ -137,12 +140,10 @@ async fn main() -> Result<(), Box<dyn Error>> {
let cache: Arc<Box<dyn Cache>> = if low_mem_mode {
DiskCache::new(disk_quota, cache_path.clone()).await
} else if use_lfu {
MemoryCache::<mem::Lfu>::new(disk_quota, cache_path.clone(), memory_max_size).await
} else {
if use_lfu {
MemoryCache::<mem::Lfu>::new(disk_quota, cache_path.clone(), memory_max_size).await
} else {
MemoryCache::<mem::Lru>::new(disk_quota, cache_path.clone(), memory_max_size).await
}
MemoryCache::<mem::Lru>::new(disk_quota, cache_path.clone(), memory_max_size).await
};
let cache_0 = Arc::clone(&cache);

View file

@ -42,7 +42,7 @@ impl<'a> Request<'a> {
.expect("to parse the build version"),
tls_created_at: TLS_PREVIOUSLY_CREATED
.get()
.map(|v| v.load().as_ref().to_owned()),
.map(|v| v.load().as_ref().clone()),
}
}
}

View file

@ -13,7 +13,7 @@ use chrono::{DateTime, Utc};
use futures::{Stream, TryStreamExt};
use log::{debug, error, info, warn};
use once_cell::sync::Lazy;
use reqwest::Client;
use reqwest::{Client, StatusCode};
use serde::Deserialize;
use sodiumoxide::crypto::box_::{open_precomputed, Nonce, PrecomputedKey, NONCEBYTES};
use thiserror::Error;
@ -238,7 +238,7 @@ async fn fetch_image(
.map(|v| String::from_utf8_lossy(v.as_ref()).contains("image/"))
.unwrap_or_default();
if resp.status() != 200 || !is_image {
if resp.status() != StatusCode::OK || !is_image {
warn!(
"Got non-OK or non-image response code from upstream, proxying and not caching result.",
);
@ -283,7 +283,7 @@ async fn fetch_image(
debug!("Done putting into cache");
return construct_response(stream, &metadata);
construct_response(stream, &metadata)
}
Err(e) => {
error!("Failed to fetch image from server: {}", e);

View file

@ -1,4 +1,5 @@
use log::{info, warn};
use reqwest::StatusCode;
use serde::Serialize;
const CONTROL_CENTER_STOP_URL: &str = "https://api.mangadex.network/ping";
@ -18,7 +19,7 @@ pub async fn send_stop(secret: &str) {
.await
{
Ok(resp) => {
if resp.status() == 200 {
if resp.status() == StatusCode::OK {
info!("Successfully sent stop message to control center.");
} else {
warn!("Got weird response from server: {:?}", resp.headers());