initial work

feature/yaml-config
Edward Shen 2021-06-16 15:31:03 -04:00
parent e33c0c73e1
commit 5b67431778
Signed by: edward
GPG Key ID: 19182661E818369F
10 changed files with 321 additions and 28 deletions

89
Cargo.lock generated
View File

@ -1,5 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "actix-codec"
version = "0.4.0"
@ -563,13 +565,14 @@ dependencies = [
[[package]]
name = "derive_more"
version = "0.99.14"
version = "0.99.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320"
checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df"
dependencies = [
"convert_case",
"proc-macro2",
"quote",
"rustc_version 0.3.3",
"syn",
]
@ -600,6 +603,12 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
[[package]]
name = "dtoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
[[package]]
name = "ed25519"
version = "1.1.1"
@ -1105,6 +1114,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linked-hash-map"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
[[package]]
name = "local-channel"
version = "0.1.2"
@ -1176,6 +1191,7 @@ dependencies = [
"serde",
"serde_json",
"serde_repr",
"serde_yaml",
"simple_logger",
"sodiumoxide",
"sqlx",
@ -1361,6 +1377,15 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pest"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
dependencies = [
"ucd-trie",
]
[[package]]
name = "pin-project"
version = "1.0.7"
@ -1628,6 +1653,15 @@ dependencies = [
"semver 0.9.0",
]
[[package]]
name = "rustc_version"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee"
dependencies = [
"semver 0.11.0",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
@ -1693,7 +1727,16 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser",
"semver-parser 0.7.0",
]
[[package]]
name = "semver"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
dependencies = [
"semver-parser 0.10.2",
]
[[package]]
@ -1708,6 +1751,15 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "semver-parser"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7"
dependencies = [
"pest",
]
[[package]]
name = "serde"
version = "1.0.126"
@ -1762,6 +1814,18 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_yaml"
version = "0.8.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15654ed4ab61726bf918a39cb8d98a2e2995b002387807fa6ba58fdf7f59bb23"
dependencies = [
"dtoa",
"linked-hash-map",
"serde",
"yaml-rust",
]
[[package]]
name = "sha-1"
version = "0.9.6"
@ -2055,9 +2119,9 @@ dependencies = [
[[package]]
name = "sysinfo"
version = "0.19.0"
version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41c72bb3368a39bb153f226d8dd426ee33d54bc31f44037c078391b4a0b8dc04"
checksum = "a68b8aff80646f09ec11e59b56091a5278ee527d5baeca938f1a5dbd9a15a859"
dependencies = [
"cfg-if",
"core-foundation-sys",
@ -2292,6 +2356,12 @@ version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
[[package]]
name = "ucd-trie"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c"
[[package]]
name = "unicode-bidi"
version = "0.3.5"
@ -2569,6 +2639,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "zstd"
version = "0.7.0+zstd.1.4.9"

View File

@ -36,6 +36,7 @@ rustls = "0.19"
serde = "1"
serde_json = "1"
serde_repr = "0.1"
serde_yaml = "0.8"
simple_logger = "1"
sodiumoxide = "0.2"
sqlx = { version = "0.5", features = [ "runtime-actix-rustls", "sqlite", "time", "chrono", "macros" ] }

14
docs/ciphers.md Normal file
View File

@ -0,0 +1,14 @@
# Ciphers
This client relies on rustls, which only supports a subset of TLS ciphers.
Specifically, only TLS 1.2 ECDSA GCM ciphers as well as all TLS 1.3 ciphers are
supported. This means that clients that only support older, more insecure
ciphers may not be able to connect to this client.
In practice, this means this client's failure rate may be higher than expected.
This is okay, and within specifications.
## Why even bother?
Well, Australia has officially banned hentai... so I gotta make sure my mates
over there won't get in trouble if I'm connecting to them.

14
docs/unstable_options.md Normal file
View File

@ -0,0 +1,14 @@
# Unstable Options
Unstable options are options that are either experimental, dangerous, for
development only, or a mix of the three. The following table describes each
option. Generally speaking, you should never need to enable these unless you
know what you're doing.
| Option | Experimental? | Dangerous? | For development? |
| -------------------------- | ------------- | ---------- | ---------------- |
| `override-upstream` | | | Yes |
| `use-lfu` | Yes | | |
| `disable-token-validation` | | Yes | Yes |
| `offline-mode` | | | Yes |
| `disable-tls` | | Yes | Yes |

61
settings.sample.yaml Normal file
View File

@ -0,0 +1,61 @@
---
# ⢸⣿⣿⣿⣿⠃⠄⢀⣴⡾⠃⠄⠄⠄⠄⠄⠈⠺⠟⠛⠛⠛⠛⠻⢿⣿⣿⣿⣿⣶⣤⡀⠄
# ⢸⣿⣿⣿⡟⢀⣴⣿⡿⠁⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⣸⣿⣿⣿⣿⣿⣿⣿⣷
# ⢸⣿⣿⠟⣴⣿⡿⡟⡼⢹⣷⢲⡶⣖⣾⣶⢄⠄⠄⠄⠄⠄⢀⣼⣿⢿⣿⣿⣿⣿⣿⣿⣿
# ⢸⣿⢫⣾⣿⡟⣾⡸⢠⡿⢳⡿⠍⣼⣿⢏⣿⣷⢄⡀⠄⢠⣾⢻⣿⣸⣿⣿⣿⣿⣿⣿⣿
# ⡿⣡⣿⣿⡟⡼⡁⠁⣰⠂⡾⠉⢨⣿⠃⣿⡿⠍⣾⣟⢤⣿⢇⣿⢇⣿⣿⢿⣿⣿⣿⣿⣿
# ⣱⣿⣿⡟⡐⣰⣧⡷⣿⣴⣧⣤⣼⣯⢸⡿⠁⣰⠟⢀⣼⠏⣲⠏⢸⣿⡟⣿⣿⣿⣿⣿⣿
# ⣿⣿⡟⠁⠄⠟⣁⠄⢡⣿⣿⣿⣿⣿⣿⣦⣼⢟⢀⡼⠃⡹⠃⡀⢸⡿⢸⣿⣿⣿⣿⣿⡟
# ⣿⣿⠃⠄⢀⣾⠋⠓⢰⣿⣿⣿⣿⣿⣿⠿⣿⣿⣾⣅⢔⣕⡇⡇⡼⢁⣿⣿⣿⣿⣿⣿⢣
# ⣿⡟⠄⠄⣾⣇⠷⣢⣿⣿⣿⣿⣿⣿⣿⣭⣀⡈⠙⢿⣿⣿⡇⡧⢁⣾⣿⣿⣿⣿⣿⢏⣾
# ⣿⡇⠄⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠟⢻⠇⠄⠄⢿⣿⡇⢡⣾⣿⣿⣿⣿⣿⣏⣼⣿
# ⣿⣷⢰⣿⣿⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⢰⣧⣀⡄⢀⠘⡿⣰⣿⣿⣿⣿⣿⣿⠟⣼⣿⣿
# ⢹⣿⢸⣿⣿⠟⠻⢿⣿⣿⣿⣿⣿⣿⣿⣶⣭⣉⣤⣿⢈⣼⣿⣿⣿⣿⣿⣿⠏⣾⣹⣿⣿
# ⢸⠇⡜⣿⡟⠄⠄⠄⠈⠙⣿⣿⣿⣿⣿⣿⣿⣿⠟⣱⣻⣿⣿⣿⣿⣿⠟⠁⢳⠃⣿⣿⣿
# ⠄⣰⡗⠹⣿⣄⠄⠄⠄⢀⣿⣿⣿⣿⣿⣿⠟⣅⣥⣿⣿⣿⣿⠿⠋⠄⠄⣾⡌⢠⣿⡿⠃
# ⠜⠋⢠⣷⢻⣿⣿⣶⣾⣿⣿⣿⣿⠿⣛⣥⣾⣿⠿⠟⠛⠉⠄⠄
#
# MangaDex@Home configuration file
# We are pleased to have you here
# May fate stay the night with you!
# The size in mebibytes of the cache
# You can use megabytes instead in a pinch,
# but just know the two are **NOT** the same.
max_cache_size_in_mebibytes: 0
server_settings:
# The client secret
# Keep this secret at all costs :P
secret: ina_is_the_cutest
# The port for the webserver to listen on
# 443 is recommended for maximum appeal
port: 443
# This controls the value the server receives
# for your upload speed
external_max_kilobits_per_second: 0
#
# Stuff that you probably don't need to change
#
# The external port to broadcast to the backend
# Keep this at 0 unless you know what you're doing
# 0 means broadcast the same value as `port`
external_port: 0
# How long to wait for the graceful shutdown (Ctrl-C or SIGINT)
# This is rounded to a multiple of 5 seconds
graceful_shutdown_wait_seconds: 60
# The external hostname to listen on
# Keep this at 0.0.0.0 unless you know what you're doing
hostname: 0.0.0.0
# The external ip to broadcast to the webserver
# The default of null means the backend will infer it
# from where it was sent from, which may fail in the
# presence of multiple IPs
external_ip: ~

View File

@ -1,12 +1,18 @@
use std::fmt::{Display, Formatter};
use std::fs::{File, OpenOptions};
use std::io::{ErrorKind, Write};
use std::net::IpAddr;
use std::num::{NonZeroU16, NonZeroU64};
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::atomic::AtomicBool;
use clap::{crate_authors, crate_description, crate_version, Clap};
use serde::{Deserialize, Serialize};
use url::Url;
use crate::units::{Kilobits, Mebibytes, Port};
// Validate tokens is an atomic because it's faster than locking on rwlock.
pub static VALIDATE_TOKENS: AtomicBool = AtomicBool::new(false);
// We use an atomic here because it's better for us to not pass the config
@ -15,41 +21,110 @@ pub static SEND_SERVER_VERSION: AtomicBool = AtomicBool::new(false);
pub static OFFLINE_MODE: AtomicBool = AtomicBool::new(false);
pub fn load_config() -> Config {
const CONFIG_PATH: &str = "./settings.yaml";
let config_file: Result<YamlArgs, _> = match File::open(CONFIG_PATH) {
Ok(file) => serde_yaml::from_reader(file),
Err(e) if e.kind() == ErrorKind::NotFound => {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(CONFIG_PATH)
.unwrap();
file.write_all(include_bytes!("../settings.sample.yaml"))
.unwrap();
return load_config();
}
_ => panic!(),
};
todo!()
}
pub struct Config {}
#[derive(Deserialize)]
struct YamlArgs {
// Naming is legacy
max_cache_size_in_mebibytes: Mebibytes,
server_settings: YamlServerSettings,
// This implementation custom options
extended_options: YamlExtendedOptions,
}
// Naming is legacy
#[derive(Deserialize)]
struct YamlServerSettings {
secret: ClientSecret,
port: Port,
external_max_kilobits_per_second: Kilobits,
external_port: Option<Port>,
graceful_shutdown_wait_seconds: Option<NonZeroU16>,
hostname: Option<IpAddr>,
external_ip: Option<IpAddr>,
}
// this intentionally does not implement display or debug
#[derive(Deserialize, Serialize)]
struct ClientSecret(String);
#[derive(Deserialize)]
struct YamlExtendedOptions {
memory_quota: Option<NonZeroU64>,
#[serde(default)]
send_server_string: bool,
#[serde(default)]
cache_type: YamlCacheType,
#[serde(default)]
ephemeral_disk_encryption: bool,
#[serde(default)]
enable_metrics: bool,
}
#[derive(Deserialize)]
enum YamlCacheType {
OnDisk,
Lru,
Lfu,
}
impl Default for YamlCacheType {
fn default() -> Self {
Self::OnDisk
}
}
#[derive(Clap, Clone)]
#[clap(version = crate_version!(), author = crate_authors!(), about = crate_description!())]
pub struct CliArgs {
/// The port to listen on.
#[clap(short, long, default_value = "42069", env = "PORT")]
pub port: NonZeroU16,
#[clap(short, long, default_value = "42069")]
pub port: Port,
/// How large, in bytes, the in-memory cache should be. Note that this does
/// not include runtime memory usage.
#[clap(long, env = "MEM_CACHE_QUOTA_BYTES", conflicts_with = "low-memory")]
#[clap(long, conflicts_with = "low-memory")]
pub memory_quota: Option<NonZeroU64>,
/// How large, in bytes, the on-disk cache should be. Note that actual
/// values may be larger for metadata information.
#[clap(long, env = "DISK_CACHE_QUOTA_BYTES")]
#[clap(long)]
pub disk_quota: u64,
/// Sets the location of the disk cache.
#[clap(long, default_value = "./cache", env = "DISK_CACHE_PATH")]
#[clap(long, default_value = "./cache")]
pub cache_path: PathBuf,
/// The network speed to advertise to Mangadex@Home control server.
#[clap(long, env = "MAX_NETWORK_SPEED")]
#[clap(long)]
pub network_speed: NonZeroU64,
/// Whether or not to provide the Server HTTP header to clients. This is
/// useful for debugging, but is generally not recommended for security
/// reasons.
#[clap(long, env = "ENABLE_SERVER_STRING", takes_value = false)]
pub enable_server_string: bool,
#[clap(long, takes_value = false)]
pub send_server_string: bool,
/// Changes the caching behavior to avoid buffering images in memory, and
/// instead use the filesystem as the buffer backing. This is useful for
/// clients in low (< 1GB) RAM environments.
#[clap(
short,
long,
conflicts_with("memory-quota"),
env = "LOW_MEMORY_MODE",
takes_value = false
)]
#[clap(short, long, conflicts_with("memory-quota"), takes_value = false)]
pub low_memory: bool,
/// Changes verbosity. Default verbosity is INFO, while increasing counts of
/// verbose flags increases the verbosity to DEBUG and TRACE, respectively.
@ -69,6 +144,8 @@ pub struct CliArgs {
/// performance, privacy, and usability with this flag enabled.
#[clap(short, long)]
pub ephemeral_disk_encryption: bool,
#[clap(short, long)]
pub config_path: Option<PathBuf>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]

View File

@ -39,6 +39,7 @@ mod ping;
mod routes;
mod state;
mod stop;
mod units;
#[macro_export]
macro_rules! client_api_version {
@ -65,6 +66,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
// Config loading
//
let config = config::load_config();
let cli_args = CliArgs::parse();
let port = cli_args.port;
let memory_max_size = cli_args
@ -132,9 +135,12 @@ async fn main() -> Result<(), Box<dyn Error>> {
let data_0 = Arc::new(RwLockServerState(RwLock::new(server)));
let data_1 = Arc::clone(&data_0);
// What's nice is that Rustls only supports TLS 1.2 and 1.3.
let mut tls_config = ServerConfig::new(NoClientAuth::new());
tls_config.cert_resolver = Arc::new(DynamicServerCert);
// Rustls only supports TLS 1.2 and 1.3.
let tls_config = {
let mut tls_config = ServerConfig::new(NoClientAuth::new());
tls_config.cert_resolver = Arc::new(DynamicServerCert);
tls_config
};
//
// At this point, the server is ready to start, and starts the necessary
@ -208,11 +214,12 @@ async fn main() -> Result<(), Box<dyn Error>> {
})
.shutdown_timeout(60);
let server_bind_address = format!("0.0.0.0:{}", port);
if disable_tls {
server.bind(format!("0.0.0.0:{}", port))?.run().await?;
server.bind(server_bind_address)?.run().await?;
} else {
server
.bind_rustls(format!("0.0.0.0:{}", port), tls_config)?
.bind_rustls(server_bind_address, tls_config)?
.run()
.await?;
}

View File

@ -1,4 +1,4 @@
use std::num::{NonZeroU16, NonZeroU64};
use std::num::NonZeroU64;
use std::sync::atomic::Ordering;
use std::{io::BufReader, sync::Arc};
@ -17,6 +17,7 @@ use crate::state::{
RwLockServerState, PREVIOUSLY_COMPROMISED, PREVIOUSLY_PAUSED, TLS_CERTS,
TLS_PREVIOUSLY_CREATED, TLS_SIGNING_KEY,
};
use crate::units::Port;
use crate::{client_api_version, config::UnstableOptions};
pub const CONTROL_CENTER_PING_URL: &str = "https://api.mangadex.network/ping";
@ -24,7 +25,7 @@ pub const CONTROL_CENTER_PING_URL: &str = "https://api.mangadex.network/ping";
#[derive(Serialize, Debug)]
pub struct Request<'a> {
secret: &'a str,
port: NonZeroU16,
port: Port,
disk_space: u64,
network_speed: NonZeroU64,
build_version: u64,

View File

@ -52,7 +52,7 @@ impl ServerState {
.send()
.await;
if config.enable_server_string {
if config.send_server_string {
warn!("Client will send Server header in responses. This is not recommended!");
SEND_SERVER_VERSION.store(true, Ordering::Release);
}

39
src/units.rs Normal file
View File

@ -0,0 +1,39 @@
use std::fmt::Display;
use std::num::{NonZeroU16, NonZeroU64};
use std::str::FromStr;
use serde::{Deserialize, Serialize};
/// Wrapper type for a port number.
#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub struct Port(NonZeroU16);
impl FromStr for Port {
type Err = <NonZeroU16 as FromStr>::Err;
fn from_str(s: &str) -> Result<Self, Self::Err> {
NonZeroU16::from_str(s).map(Self)
}
}
impl Display for Port {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
#[derive(Copy, Clone, Deserialize, Default, Debug, Hash, Eq, PartialEq)]
pub struct Mebibytes(usize);
impl Mebibytes {
pub const fn as_bytes(&self) -> usize {
self.0 << 20
}
pub const fn get(&self) -> usize {
self.0
}
}
#[derive(Copy, Clone, Deserialize, Debug, Hash, Eq, PartialEq)]
pub struct Kilobits(NonZeroU64);