edward/mangadex-home-rs
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Compare commits

base: edward:5fd8e86d9739c63ae0db46e99b8ef6fa1ddadba7
Branches Tags
edward:master
edward:feature/yaml-config
edward:feature/v32-tokens
..
compare: edward:96a6b76baa9a47e3dc8c87a752935a810faa4949
Branches Tags
edward:master
edward:feature/yaml-config
edward:feature/v32-tokens

No commits in common. "5fd8e86d9739c63ae0db46e99b8ef6fa1ddadba7" and "96a6b76baa9a47e3dc8c87a752935a810faa4949" have entirely different histories.
5fd8e86d97 ... 96a6b76baa

7 changed files with 32 additions and 229 deletions
Show stats Expand all files Collapse all files

38
Cargo.lock generated
View file

@ -1051,7 +1051,6 @@ dependencies = [
"lru",
"once_cell",
"parking_lot",
"prometheus",
"reqwest",
"rustls",
"serde",
@ -1330,43 +1329,6 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "procfs"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab8809e0c18450a2db0f236d2a44ec0b4c1412d0eb936233579f0990faa5d5cd"
dependencies = [
"bitflags",
"byteorder",
"flate2",
"hex",
"lazy_static",
"libc",
]
[[package]]
name = "prometheus"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c"
dependencies = [
"cfg-if",
"fnv",
"lazy_static",
"libc",
"memchr",
"parking_lot",
"procfs",
"protobuf",
"thiserror",
]
[[package]]
name = "protobuf"
version = "2.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45604fc7a88158e7d514d8e22e14ac746081e7a70d7690074dd0029ee37458d6"
[[package]]
name = "quote"
version = "1.0.9"

1
Cargo.toml
View file

@ -30,7 +30,6 @@ log = "0.4"
lfu_cache = "1"
lru = "0.6"
parking_lot = "0.11"
prometheus = { version = "0.12", features = [ "process" ] }
reqwest = { version = "0.11", default_features = false, features = [ "json", "stream", "rustls-tls" ] }
rustls = "0.19"
serde = "1"

16
src/config.rs
View file

@ -13,8 +13,6 @@ pub static VALIDATE_TOKENS: AtomicBool = AtomicBool::new(false);
// everywhere.
pub static SEND_SERVER_VERSION: AtomicBool = AtomicBool::new(false);
pub static OFFLINE_MODE: AtomicBool = AtomicBool::new(false);
#[derive(Clap, Clone)]
#[clap(version = crate_version!(), author = crate_authors!(), about = crate_description!())]
pub struct CliArgs {
@ -84,25 +82,17 @@ pub enum UnstableOptions {
/// Disables token validation. Don't use this unless you know the
/// ramifications of this command.
DisableTokenValidation,
/// Tries to run without communication to MangaDex.
OfflineMode,
/// Serves HTTP in plaintext
DisableTls,
}
impl FromStr for UnstableOptions {
type Err = String;
type Err = &'static str;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"override-upstream" => Ok(Self::OverrideUpstream),
"use-lfu" => Ok(Self::UseLfu),
"disable-token-validation" => Ok(Self::DisableTokenValidation),
"offline-mode" => Ok(Self::OfflineMode),
"disable-tls" => Ok(Self::DisableTls),
_ => Err(format!("Unknown unstable option '{}'", s)),
_ => Err("Unknown unstable option"),
}
}
}
@ -113,8 +103,6 @@ impl Display for UnstableOptions {
Self::OverrideUpstream => write!(f, "override-upstream"),
Self::UseLfu => write!(f, "use-lfu"),
Self::DisableTokenValidation => write!(f, "disable-token-validation"),
Self::OfflineMode => write!(f, "offline-mode"),
Self::DisableTls => write!(f, "disable-tls"),
}
}
}

58
src/main.rs
View file

@ -29,12 +29,11 @@ use thiserror::Error;
use crate::cache::mem::{Lfu, Lru};
use crate::cache::{MemoryCache, ENCRYPTION_KEY};
use crate::config::{UnstableOptions, OFFLINE_MODE};
use crate::config::UnstableOptions;
use crate::state::DynamicServerCert;
mod cache;
mod config;
mod metrics;
mod ping;
mod routes;
mod state;
@ -60,12 +59,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
sodiumoxide::init().expect("Failed to initialize crypto");
// It's ok to fail early here, it would imply we have a invalid config.
dotenv::dotenv().ok();
//
// Config loading
//
let cli_args = CliArgs::parse();
let port = cli_args.port;
let memory_max_size = cli_args
.memory_quota
@ -75,19 +70,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
let cache_path = cli_args.cache_path.clone();
let low_mem_mode = cli_args.low_memory;
let use_lfu = cli_args.unstable_options.contains(&UnstableOptions::UseLfu);
let disable_tls = cli_args
.unstable_options
.contains(&UnstableOptions::DisableTls);
OFFLINE_MODE.store(
cli_args
.unstable_options
.contains(&UnstableOptions::OfflineMode),
Ordering::Release,
);
//
// Logging and warnings
//
let log_level = match (cli_args.quiet, cli_args.verbose) {
(n, _) if n > 2 => LevelFilter::Off,
@ -120,15 +102,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
ENCRYPTION_KEY.set(gen_key()).unwrap();
}
metrics::init();
// HTTP Server init
let server = if OFFLINE_MODE.load(Ordering::Acquire) {
ServerState::init_offline()
} else {
ServerState::init(&client_secret, &cli_args).await?
};
let server = ServerState::init(&client_secret, &cli_args).await?;
let data_0 = Arc::new(RwLockServerState(RwLock::new(server)));
let data_1 = Arc::clone(&data_0);
@ -149,7 +123,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
let system = &system;
let client_secret = client_secret.clone();
let running_2 = Arc::clone(&running_1);
if !OFFLINE_MODE.load(Ordering::Acquire) {
System::new().block_on(async move {
if running_2.load(Ordering::SeqCst) {
send_stop(&client_secret).await;
@ -158,13 +131,11 @@ async fn main() -> Result<(), Box<dyn Error>> {
system.stop()
}
});
}
running_1.store(false, Ordering::SeqCst);
})
.expect("Error setting Ctrl-C handler");
// Spawn ping task
if !OFFLINE_MODE.load(Ordering::Acquire) {
spawn(async move {
let mut interval = time::interval(Duration::from_secs(90));
let mut data = Arc::clone(&data_0);
@ -174,7 +145,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
ping::update_server_state(&client_secret_1, &cli_args, &mut data).await;
}
});
}
let cache = DiskCache::new(disk_quota, cache_path.clone()).await;
let cache: Arc<dyn Cache> = if low_mem_mode {
@ -188,25 +158,18 @@ async fn main() -> Result<(), Box<dyn Error>> {
let cache_0 = Arc::clone(&cache);
// Start HTTPS server
let server = HttpServer::new(move || {
HttpServer::new(move || {
App::new()
.service(routes::token_data)
.service(routes::token_data_saver)
.service(routes::metrics)
.route("{tail:.*}", web::get().to(routes::default))
.app_data(Data::from(Arc::clone(&data_1)))
.app_data(Data::from(Arc::clone(&cache_0)))
})
.shutdown_timeout(60);
if disable_tls {
server.bind(format!("0.0.0.0:{}", port))?.run().await?;
} else {
server
.shutdown_timeout(60)
.bind_rustls(format!("0.0.0.0:{}", port), tls_config)?
.run()
.await?;
}
// Waiting for us to finish sending stop message
while running.load(Ordering::SeqCst) {
@ -265,17 +228,6 @@ fn print_preamble_and_warnings(args: &CliArgs) -> Result<(), Box<dyn Error>> {
warn!("Unstable options are enabled. These options should not be used in production!");
}
if args
.unstable_options
.contains(&UnstableOptions::OfflineMode)
{
warn!("Running in offline mode. No communication to MangaDex will be made!");
}
if args.unstable_options.contains(&UnstableOptions::DisableTls) {
warn!("Serving insecure traffic! You better be running this for development only.");
}
if args.override_upstream.is_some()
&& !args
.unstable_options

45
src/metrics.rs
View file

@ -1,45 +0,0 @@
use once_cell::sync::Lazy;
use prometheus::{register_int_counter, IntCounter};
pub static CACHE_HIT_COUNTER: Lazy<IntCounter> =
Lazy::new(|| register_int_counter!("cache_hit", "The number of cache hits").unwrap());
pub static CACHE_MISS_COUNTER: Lazy<IntCounter> =
Lazy::new(|| register_int_counter!("cache_miss", "The number of cache misses").unwrap());
pub static REQUESTS_TOTAL_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!("requests_total", "The total number of requests served.").unwrap()
});
pub static REQUESTS_DATA_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!(
"requests_data",
"The number of requests served from the /data endpoint."
)
.unwrap()
});
pub static REQUESTS_DATA_SAVER_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!(
"requests_data_saver",
"The number of requests served from the /data-saver endpoint."
)
.unwrap()
});
pub static REQUESTS_OTHER_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
register_int_counter!(
"requests_other",
"The total number of request not served by primary endpoints."
)
.unwrap()
});
pub fn init() {
let _a = CACHE_HIT_COUNTER.get();
let _a = CACHE_MISS_COUNTER.get();
let _a = REQUESTS_TOTAL_COUNTER.get();
let _a = REQUESTS_DATA_COUNTER.get();
let _a = REQUESTS_DATA_SAVER_COUNTER.get();
let _a = REQUESTS_OTHER_COUNTER.get();
}

46
src/routes.rs
View file

@ -1,6 +1,5 @@
use std::sync::atomic::Ordering;
use actix_web::error::ErrorNotFound;
use actix_web::http::header::{
ACCESS_CONTROL_ALLOW_ORIGIN, ACCESS_CONTROL_EXPOSE_HEADERS, CACHE_CONTROL, CONTENT_LENGTH,
CONTENT_TYPE, LAST_MODIFIED, X_CONTENT_TYPE_OPTIONS,
@ -14,7 +13,6 @@ use chrono::{DateTime, Utc};
use futures::{Stream, TryStreamExt};
use log::{debug, error, info, warn};
use once_cell::sync::Lazy;
use prometheus::{Encoder, TextEncoder};
use reqwest::{Client, StatusCode};
use serde::Deserialize;
use sodiumoxide::crypto::box_::{open_precomputed, Nonce, PrecomputedKey, NONCEBYTES};
@ -22,11 +20,7 @@ use thiserror::Error;
use crate::cache::{Cache, CacheKey, ImageMetadata, UpstreamError};
use crate::client_api_version;
use crate::config::{OFFLINE_MODE, SEND_SERVER_VERSION, VALIDATE_TOKENS};
use crate::metrics::{
CACHE_HIT_COUNTER, CACHE_MISS_COUNTER, REQUESTS_DATA_COUNTER, REQUESTS_DATA_SAVER_COUNTER,
REQUESTS_OTHER_COUNTER, REQUESTS_TOTAL_COUNTER,
};
use crate::config::{SEND_SERVER_VERSION, VALIDATE_TOKENS};
use crate::state::RwLockServerState;
pub const BASE64_CONFIG: base64::Config = base64::Config::new(base64::CharacterSet::UrlSafe, false);
@ -52,10 +46,7 @@ impl Responder for ServerResponse {
fn respond_to(self, req: &HttpRequest) -> HttpResponse {
match self {
Self::TokenValidationError(e) => e.respond_to(req),
Self::HttpResponse(resp) => {
REQUESTS_TOTAL_COUNTER.inc();
resp.respond_to(req)
}
Self::HttpResponse(resp) => resp.respond_to(req),
}
}
}
@ -67,7 +58,6 @@ async fn token_data(
cache: Data<dyn Cache>,
path: Path<(String, String, String)>,
) -> impl Responder {
REQUESTS_DATA_COUNTER.inc();
let (token, chapter_hash, file_name) = path.into_inner();
if VALIDATE_TOKENS.load(Ordering::Acquire) {
if let Err(e) = validate_token(&state.0.read().precomputed_key, token, &chapter_hash) {
@ -84,7 +74,6 @@ async fn token_data_saver(
cache: Data<dyn Cache>,
path: Path<(String, String, String)>,
) -> impl Responder {
REQUESTS_DATA_SAVER_COUNTER.inc();
let (token, chapter_hash, file_name) = path.into_inner();
if VALIDATE_TOKENS.load(Ordering::Acquire) {
if let Err(e) = validate_token(&state.0.read().precomputed_key, token, &chapter_hash) {
@ -97,22 +86,12 @@ async fn token_data_saver(
#[allow(clippy::future_not_send)]
pub async fn default(state: Data<RwLockServerState>, req: HttpRequest) -> impl Responder {
REQUESTS_OTHER_COUNTER.inc();
let path = &format!(
"{}{}",
state.0.read().image_server,
req.path().chars().skip(1).collect::<String>()
);
if OFFLINE_MODE.load(Ordering::Acquire) {
info!("Got unknown path in offline mode, returning 404: {}", path);
return ServerResponse::HttpResponse(
ErrorNotFound("Path is not valid in offline mode").into(),
);
} else {
info!("Got unknown path, just proxying: {}", path);
}
let resp = match HTTP_CLIENT.get(path).send().await {
Ok(resp) => resp,
Err(e) => {
@ -130,17 +109,6 @@ pub async fn default(state: Data<RwLockServerState>, req: HttpRequest) -> impl R
ServerResponse::HttpResponse(resp_builder.body(resp.bytes().await.unwrap_or_default()))
}
#[allow(clippy::future_not_send)]
#[get("/metrics")]
pub async fn metrics() -> impl Responder {
let metric_families = prometheus::gather();
let mut buffer = Vec::new();
TextEncoder::new()
.encode(&metric_families, &mut buffer)
.unwrap();
String::from_utf8(buffer).unwrap()
}
#[derive(Error, Debug)]
enum TokenValidationError {
#[error("Failed to decode base64 token.")]
@ -230,7 +198,6 @@ async fn fetch_image(
match cache.get(&key).await {
Some(Ok((image, metadata))) => {
CACHE_HIT_COUNTER.inc();
return construct_response(image, &metadata);
}
Some(Err(_)) => {
@ -239,15 +206,6 @@ async fn fetch_image(
_ => (),
}
CACHE_MISS_COUNTER.inc();
// If in offline mode, return early since there's nothing else we can do
if OFFLINE_MODE.load(Ordering::Acquire) {
return ServerResponse::HttpResponse(
ErrorNotFound("Offline mode enabled and image not in cache").into(),
);
}
// It's important to not get a write lock before this request, else we're
// holding the read lock until the await resolves.

15
src/state.rs
View file

@ -1,7 +1,6 @@
use std::str::FromStr;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::config::{CliArgs, UnstableOptions, OFFLINE_MODE, SEND_SERVER_VERSION, VALIDATE_TOKENS};
use crate::config::{CliArgs, UnstableOptions, SEND_SERVER_VERSION, VALIDATE_TOKENS};
use crate::ping::{Request, Response, CONTROL_CENTER_PING_URL};
use arc_swap::ArcSwap;
use log::{error, info, warn};
@ -10,7 +9,7 @@ use parking_lot::RwLock;
use rustls::sign::{CertifiedKey, SigningKey};
use rustls::Certificate;
use rustls::{ClientHello, ResolvesServerCert};
use sodiumoxide::crypto::box_::{PrecomputedKey, PRECOMPUTEDKEYBYTES};
use sodiumoxide::crypto::box_::PrecomputedKey;
use thiserror::Error;
use url::Url;
@ -145,16 +144,6 @@ impl ServerState {
},
}
}
pub fn init_offline() -> Self {
assert!(OFFLINE_MODE.load(Ordering::Acquire));
Self {
precomputed_key: PrecomputedKey::from_slice(&[41; PRECOMPUTEDKEYBYTES]).unwrap(),
image_server: Url::from_file_path("/dev/null").unwrap(),
url: Url::from_str("http://localhost").unwrap(),
url_overridden: false,
}
}
}
pub struct RwLockServerState(pub RwLock<ServerState>);