Compare commits

..

No commits in common. "c5383639f5d76d016291bb106389724c8a2f5d90" and "a8e5d09ff0fa5a5b382b6d6b988318bf8968c460" have entirely different histories.

8 changed files with 78 additions and 164 deletions

1
.gitignore vendored
View file

@ -4,4 +4,3 @@
flamegraph*.svg
perf.data*
dhat.out.*
settings.yaml

View file

@ -1,25 +1,10 @@
use std::error::Error;
use std::process::Command;
use vergen::{vergen, Config, ShaKind};
fn main() -> Result<(), Box<dyn Error>> {
// Initialize vergen stuff
let mut config = Config::default();
*config.git_mut().sha_kind_mut() = ShaKind::Short;
vergen(config)?;
// Initialize SQL stuff
let project_root = std::env::var("CARGO_MANIFEST_DIR").unwrap();
Command::new("mkdir")
.args(["cache", "--parents"])
.current_dir(&project_root)
.output()?;
Command::new("sqlite3")
.args(["cache/metadata.sqlite", include_str!("db_queries/init.sql")])
.current_dir(&project_root)
.output()?;
Ok(())
}

10
init_cache.sh Executable file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
# This script needs to be run once in order for compile time macros to not
# complain about a missing DB
# We can trust that our program will initialize the db at runtime the same way
# as it pulls from the same file for initialization
mkdir cache
sqlite3 cache/metadata.sqlite < db_queries/init.sql

View file

@ -18,8 +18,6 @@
# MangaDex@Home configuration file
# We are pleased to have you here
# May fate stay the night with you!
#
# Default values are commented out.
# The size in mebibytes of the cache
# You can use megabytes instead in a pinch,
@ -27,31 +25,37 @@
max_cache_size_in_mebibytes: 0
server_settings:
# The client secret. Keep this secret at all costs :P
secret: suichan wa kyou mo kawaii!
# The client secret
# Keep this secret at all costs :P
secret: ina_is_the_cutest
# The port for the webserver to listen on. 443 is recommended for max appeal.
# port: 443
# The port for the webserver to listen on
# 443 is recommended for maximum appeal
port: 443
# This controls the value the server receives for your upload speed.
# This controls the value the server receives
# for your upload speed
external_max_kilobits_per_second: 0
#
# Advanced settings
# Stuff that you probably don't need to change
#
# The external hostname to listen on. Keep this at 0.0.0.0 unless you know
# what you're doing.
# hostname: 0.0.0.0
# The external port to broadcast to the backend
# Keep this at 0 unless you know what you're doing
# 0 means broadcast the same value as `port`
external_port: 0
# The external port to broadcast to the backend. Keep this at 0 unless you
# know what you're doing. 0 means broadcast the same value as `port`.
# external_port: 0
# How long to wait for the graceful shutdown (Ctrl-C or SIGINT)
# This is rounded to a multiple of 5 seconds
graceful_shutdown_wait_seconds: 60
# How long to wait at most for the graceful shutdown (Ctrl-C or SIGINT).
# graceful_shutdown_wait_seconds: 60
# The external hostname to listen on
# Keep this at 0.0.0.0 unless you know what you're doing
hostname: 0.0.0.0
# The external ip to broadcast to the webserver. The default of null (~) means
# the backend will infer it from where it was sent from, which may fail in the
# presence of multiple IPs.
# external_ip: ~
# The external ip to broadcast to the webserver
# The default of null means the backend will infer it
# from where it was sent from, which may fail in the
# presence of multiple IPs
external_ip: ~

6
src/cache/fs.rs vendored
View file

@ -111,21 +111,21 @@ pub(super) async fn read_file(
return None;
}
let file_header = if let Some(header) = Header::from_slice(&header_bytes) {
let header = if let Some(header) = Header::from_slice(&header_bytes) {
header
} else {
warn!("Found file, but encrypted header was invalid. Assuming corrupted!");
return None;
};
let secret_stream = if let Ok(stream) = SecretStream::init_pull(&file_header, key) {
let secret_stream = if let Ok(stream) = SecretStream::init_pull(&header, key) {
stream
} else {
warn!("Failed to init secret stream with key and header. Assuming corrupted!");
return None;
};
maybe_header = Some(file_header);
maybe_header = Some(header);
reader = Some(Box::pin(EncryptedDiskReader::new(file, secret_stream)));
}

View file

@ -2,8 +2,8 @@ use std::fmt::{Display, Formatter};
use std::fs::{File, OpenOptions};
use std::hint::unreachable_unchecked;
use std::io::{ErrorKind, Write};
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::num::NonZeroU16;
use std::net::{IpAddr, SocketAddr};
use std::num::{NonZeroU16, NonZeroU64};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::atomic::{AtomicBool, Ordering};
@ -64,7 +64,7 @@ pub fn load_config() -> Result<Config, serde_yaml::Error> {
Ok(config)
}
/// Represents a fully parsed config, from a variety of sources.
/// Represents a fully parsed config file.
pub struct Config {
pub cache_type: CacheType,
pub cache_path: PathBuf,
@ -75,93 +75,27 @@ pub struct Config {
pub bind_address: SocketAddr,
pub external_address: Option<SocketAddr>,
pub ephemeral_disk_encryption: bool,
pub unstable_options: Vec<UnstableOptions>,
pub network_speed: KilobitsPerSecond,
pub disk_quota: Mebibytes,
pub memory_quota: Mebibytes,
pub unstable_options: Vec<UnstableOptions>,
pub override_upstream: Option<Url>,
pub enable_metrics: bool,
}
impl Config {
fn from_cli_and_file(cli_args: CliArgs, file_args: YamlArgs) -> Self {
let file_extended_options = file_args.extended_options.unwrap_or_default();
let log_level = match (cli_args.quiet, cli_args.verbose) {
(n, _) if n > 2 => LevelFilter::Off,
(2, _) => LevelFilter::Error,
(1, _) => LevelFilter::Warn,
// Use log level from file if no flags were provided to CLI
(0, 0) => file_extended_options
.logging_level
.unwrap_or(LevelFilter::Info),
(0, 0) => file_args.extended_options.logging_level,
(_, 1) => LevelFilter::Debug,
(_, n) if n > 1 => LevelFilter::Trace,
// compiler can't figure it out
_ => unsafe { unreachable_unchecked() },
};
let bind_port = cli_args
.port
.unwrap_or(file_args.server_settings.port)
.get();
// This needs to be outside because rust isn't smart enough yet to
// realize a disjointed borrow of a moved value is ok. This will be
// fixed in Rust 2021.
let external_port = file_args
.server_settings
.external_port
.map_or(bind_port, Port::get);
Self {
cache_type: cli_args
.cache_type
.or(file_extended_options.cache_type)
.unwrap_or_default(),
cache_path: cli_args
.cache_path
.or(file_extended_options.cache_path)
.unwrap_or_else(|| PathBuf::from_str("./cache").unwrap()),
shutdown_timeout: file_args
.server_settings
.graceful_shutdown_wait_seconds
.unwrap_or(unsafe { NonZeroU16::new_unchecked(60) }),
log_level,
// secret should never be in CLI
client_secret: file_args.server_settings.secret,
port: cli_args.port.unwrap_or(file_args.server_settings.port),
bind_address: SocketAddr::new(
file_args
.server_settings
.hostname
.unwrap_or_else(|| IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))),
bind_port,
),
external_address: file_args
.server_settings
.external_ip
.map(|ip_addr| SocketAddr::new(ip_addr, external_port)),
ephemeral_disk_encryption: cli_args
.ephemeral_disk_encryption
.or(file_extended_options.ephemeral_disk_encryption)
.unwrap_or_default(),
network_speed: cli_args
.network_speed
.unwrap_or(file_args.server_settings.external_max_kilobits_per_second),
disk_quota: cli_args
.disk_quota
.unwrap_or(file_args.max_cache_size_in_mebibytes),
memory_quota: cli_args
.memory_quota
.or(file_extended_options.memory_quota)
.unwrap_or_default(),
enable_metrics: file_extended_options.enable_metrics.unwrap_or_default(),
// Unstable options (and related) should never be in yaml config
unstable_options: cli_args.unstable_options,
override_upstream: cli_args.override_upstream,
}
todo!()
}
}
@ -171,14 +105,13 @@ struct YamlArgs {
max_cache_size_in_mebibytes: Mebibytes,
server_settings: YamlServerSettings,
// This implementation custom options
extended_options: Option<YamlExtendedOptions>,
extended_options: YamlExtendedOptions,
}
// Naming is legacy
#[derive(Deserialize)]
struct YamlServerSettings {
secret: ClientSecret,
#[serde(default)]
port: Port,
external_max_kilobits_per_second: KilobitsPerSecond,
external_port: Option<Port>,
@ -191,14 +124,21 @@ struct YamlServerSettings {
#[derive(Deserialize, Serialize)]
pub struct ClientSecret(String);
#[derive(Deserialize, Default)]
#[derive(Deserialize)]
struct YamlExtendedOptions {
memory_quota: Option<Mebibytes>,
cache_type: Option<CacheType>,
ephemeral_disk_encryption: Option<bool>,
enable_metrics: Option<bool>,
logging_level: Option<LevelFilter>,
cache_path: Option<PathBuf>,
memory_quota: Option<NonZeroU64>,
#[serde(default)]
cache_type: CacheType,
#[serde(default)]
ephemeral_disk_encryption: bool,
#[serde(default)]
enable_metrics: bool,
#[serde(default = "default_logging_level")]
logging_level: LevelFilter,
}
const fn default_logging_level() -> LevelFilter {
LevelFilter::Info
}
#[derive(Deserialize, Copy, Clone)]
@ -232,25 +172,25 @@ impl Default for CacheType {
#[clap(version = crate_version!(), author = crate_authors!(), about = crate_description!())]
struct CliArgs {
/// The port to listen on.
#[clap(short, long)]
pub port: Option<Port>,
/// How large, in mebibytes, the in-memory cache should be. Note that this
/// does not include runtime memory usage.
#[clap(short, long, default_value = "42069")]
pub port: Port,
/// How large, in bytes, the in-memory cache should be. Note that this does
/// not include runtime memory usage.
#[clap(long)]
pub memory_quota: Option<Mebibytes>,
/// How large, in mebibytes, the on-disk cache should be. Note that actual
pub memory_quota: Option<NonZeroU64>,
/// How large, in bytes, the on-disk cache should be. Note that actual
/// values may be larger for metadata information.
#[clap(long)]
pub disk_quota: Option<Mebibytes>,
pub disk_quota: u64,
/// Sets the location of the disk cache.
#[clap(long)]
pub cache_path: Option<PathBuf>,
#[clap(long, default_value = "./cache")]
pub cache_path: PathBuf,
/// The network speed to advertise to Mangadex@Home control server.
#[clap(long)]
pub network_speed: Option<KilobitsPerSecond>,
pub network_speed: NonZeroU64,
/// Changes verbosity. Default verbosity is INFO, while increasing counts of
/// verbose flags increases the verbosity to DEBUG and TRACE, respectively.
#[clap(short, long, parse(from_occurrences), conflicts_with = "quiet")]
#[clap(short, long, parse(from_occurrences))]
pub verbose: usize,
/// Changes verbosity. Default verbosity is INFO, while increasing counts of
/// quiet flags decreases the verbosity to WARN, ERROR, and no logs,
@ -265,11 +205,11 @@ struct CliArgs {
/// encrypted with a key generated at runtime. There are implications to
/// performance, privacy, and usability with this flag enabled.
#[clap(short, long)]
pub ephemeral_disk_encryption: Option<bool>,
pub ephemeral_disk_encryption: bool,
#[clap(short, long)]
pub config_path: Option<PathBuf>,
#[clap(short = 't', long)]
pub cache_type: Option<CacheType>,
#[clap(default_value = "on_disk")]
pub cache_type: CacheType,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]

View file

@ -93,9 +93,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
ENCRYPTION_KEY.set(gen_key()).unwrap();
}
if config.enable_metrics {
metrics::init();
}
metrics::init();
// HTTP Server init

View file

@ -1,5 +1,5 @@
use std::fmt::Display;
use std::num::{NonZeroU16, NonZeroU64, ParseIntError};
use std::num::{NonZeroU16, NonZeroU64};
use std::str::FromStr;
use serde::{Deserialize, Serialize};
@ -8,18 +8,6 @@ use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub struct Port(NonZeroU16);
impl Port {
pub const fn get(self) -> u16 {
self.0.get()
}
}
impl Default for Port {
fn default() -> Self {
Self(unsafe { NonZeroU16::new_unchecked(443) })
}
}
impl FromStr for Port {
type Err = <NonZeroU16 as FromStr>::Err;
@ -37,11 +25,9 @@ impl Display for Port {
#[derive(Copy, Clone, Serialize, Deserialize, Default, Debug, Hash, Eq, PartialEq)]
pub struct Mebibytes(usize);
impl FromStr for Mebibytes {
type Err = ParseIntError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
s.parse::<usize>().map(Self)
impl Mebibytes {
pub const fn get(self) -> usize {
self.0
}
}
@ -62,14 +48,6 @@ impl From<Mebibytes> for Bytes {
#[derive(Copy, Clone, Deserialize, Debug, Hash, Eq, PartialEq)]
pub struct KilobitsPerSecond(NonZeroU64);
impl FromStr for KilobitsPerSecond {
type Err = ParseIntError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
s.parse::<NonZeroU64>().map(Self)
}
}
#[derive(Copy, Clone, Serialize, Debug, Hash, Eq, PartialEq)]
pub struct BytesPerSecond(NonZeroU64);