Add support for offline mode

Cargo.lock

@@ -1051,6 +1051,7 @@ dependencies = [
  "lru",
  "once_cell",
  "parking_lot",
+ "prometheus",
  "reqwest",
  "rustls",
  "serde",
@@ -1329,6 +1330,43 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "procfs"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab8809e0c18450a2db0f236d2a44ec0b4c1412d0eb936233579f0990faa5d5cd"
+dependencies = [
+ "bitflags",
+ "byteorder",
+ "flate2",
+ "hex",
+ "lazy_static",
+ "libc",
+]
+
+[[package]]
+name = "prometheus"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c"
+dependencies = [
+ "cfg-if",
+ "fnv",
+ "lazy_static",
+ "libc",
+ "memchr",
+ "parking_lot",
+ "procfs",
+ "protobuf",
+ "thiserror",
+]
+
+[[package]]
+name = "protobuf"
+version = "2.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45604fc7a88158e7d514d8e22e14ac746081e7a70d7690074dd0029ee37458d6"
+
 [[package]]
 name = "quote"
 version = "1.0.9"

Cargo.toml

@@ -30,6 +30,7 @@ log = "0.4"
 lfu_cache = "1"
 lru = "0.6"
 parking_lot = "0.11"
+prometheus = { version = "0.12", features = [ "process" ] }
 reqwest = { version = "0.11", default_features = false, features = [ "json", "stream", "rustls-tls" ] }
 rustls = "0.19"
 serde = "1"

src/config.rs

@@ -13,6 +13,8 @@ pub static VALIDATE_TOKENS: AtomicBool = AtomicBool::new(false);
 // everywhere.
 pub static SEND_SERVER_VERSION: AtomicBool = AtomicBool::new(false);
 
+pub static OFFLINE_MODE: AtomicBool = AtomicBool::new(false);
+
 #[derive(Clap, Clone)]
 #[clap(version = crate_version!(), author = crate_authors!(), about = crate_description!())]
 pub struct CliArgs {
@@ -82,17 +84,25 @@ pub enum UnstableOptions {
     /// Disables token validation. Don't use this unless you know the
     /// ramifications of this command.
     DisableTokenValidation,
+
+    /// Tries to run without communication to MangaDex.
+    OfflineMode,
+
+    /// Serves HTTP in plaintext
+    DisableTls,
 }
 
 impl FromStr for UnstableOptions {
-    type Err = &'static str;
+    type Err = String;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "override-upstream" => Ok(Self::OverrideUpstream),
             "use-lfu" => Ok(Self::UseLfu),
             "disable-token-validation" => Ok(Self::DisableTokenValidation),
-            _ => Err("Unknown unstable option"),
+            "offline-mode" => Ok(Self::OfflineMode),
+            "disable-tls" => Ok(Self::DisableTls),
+            _ => Err(format!("Unknown unstable option '{}'", s)),
         }
     }
 }
@@ -103,6 +113,8 @@ impl Display for UnstableOptions {
             Self::OverrideUpstream => write!(f, "override-upstream"),
             Self::UseLfu => write!(f, "use-lfu"),
             Self::DisableTokenValidation => write!(f, "disable-token-validation"),
+            Self::OfflineMode => write!(f, "offline-mode"),
+            Self::DisableTls => write!(f, "disable-tls"),
         }
     }
 }

src/main.rs

@@ -29,11 +29,12 @@ use thiserror::Error;
 
 use crate::cache::mem::{Lfu, Lru};
 use crate::cache::{MemoryCache, ENCRYPTION_KEY};
-use crate::config::UnstableOptions;
+use crate::config::{UnstableOptions, OFFLINE_MODE};
 use crate::state::DynamicServerCert;
 
 mod cache;
 mod config;
+mod metrics;
 mod ping;
 mod routes;
 mod state;
@@ -59,8 +60,12 @@ async fn main() -> Result<(), Box<dyn Error>> {
     sodiumoxide::init().expect("Failed to initialize crypto");
     // It's ok to fail early here, it would imply we have a invalid config.
     dotenv::dotenv().ok();
-    let cli_args = CliArgs::parse();
 
+    //
+    // Config loading
+    //
+
+    let cli_args = CliArgs::parse();
     let port = cli_args.port;
     let memory_max_size = cli_args
         .memory_quota
@@ -70,6 +75,19 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let cache_path = cli_args.cache_path.clone();
     let low_mem_mode = cli_args.low_memory;
     let use_lfu = cli_args.unstable_options.contains(&UnstableOptions::UseLfu);
+    let disable_tls = cli_args
+        .unstable_options
+        .contains(&UnstableOptions::DisableTls);
+    OFFLINE_MODE.store(
+        cli_args
+            .unstable_options
+            .contains(&UnstableOptions::OfflineMode),
+        Ordering::Release,
+    );
+
+    //
+    // Logging and warnings
+    //
 
     let log_level = match (cli_args.quiet, cli_args.verbose) {
         (n, _) if n > 2 => LevelFilter::Off,
@@ -102,7 +120,15 @@ async fn main() -> Result<(), Box<dyn Error>> {
         ENCRYPTION_KEY.set(gen_key()).unwrap();
     }
 
-    let server = ServerState::init(&client_secret, &cli_args).await?;
+    metrics::init();
+
+    // HTTP Server init
+
+    let server = if OFFLINE_MODE.load(Ordering::Acquire) {
+        ServerState::init_offline()
+    } else {
+        ServerState::init(&client_secret, &cli_args).await?
+    };
     let data_0 = Arc::new(RwLockServerState(RwLock::new(server)));
     let data_1 = Arc::clone(&data_0);
 
@@ -123,6 +149,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
         let system = &system;
         let client_secret = client_secret.clone();
         let running_2 = Arc::clone(&running_1);
+        if !OFFLINE_MODE.load(Ordering::Acquire) {
             System::new().block_on(async move {
                 if running_2.load(Ordering::SeqCst) {
                     send_stop(&client_secret).await;
@@ -131,11 +158,13 @@ async fn main() -> Result<(), Box<dyn Error>> {
                     system.stop()
                 }
             });
+        }
         running_1.store(false, Ordering::SeqCst);
     })
     .expect("Error setting Ctrl-C handler");
 
     // Spawn ping task
+    if !OFFLINE_MODE.load(Ordering::Acquire) {
         spawn(async move {
             let mut interval = time::interval(Duration::from_secs(90));
             let mut data = Arc::clone(&data_0);
@@ -145,6 +174,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
                 ping::update_server_state(&client_secret_1, &cli_args, &mut data).await;
             }
         });
+    }
 
     let cache = DiskCache::new(disk_quota, cache_path.clone()).await;
     let cache: Arc<dyn Cache> = if low_mem_mode {
@@ -158,18 +188,25 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let cache_0 = Arc::clone(&cache);
 
     // Start HTTPS server
-    HttpServer::new(move || {
+    let server = HttpServer::new(move || {
         App::new()
             .service(routes::token_data)
             .service(routes::token_data_saver)
+            .service(routes::metrics)
             .route("{tail:.*}", web::get().to(routes::default))
             .app_data(Data::from(Arc::clone(&data_1)))
             .app_data(Data::from(Arc::clone(&cache_0)))
     })
-    .shutdown_timeout(60)
+    .shutdown_timeout(60);
+
+    if disable_tls {
+        server.bind(format!("0.0.0.0:{}", port))?.run().await?;
+    } else {
+        server
             .bind_rustls(format!("0.0.0.0:{}", port), tls_config)?
             .run()
             .await?;
+    }
 
     // Waiting for us to finish sending stop message
     while running.load(Ordering::SeqCst) {
@@ -228,6 +265,17 @@ fn print_preamble_and_warnings(args: &CliArgs) -> Result<(), Box<dyn Error>> {
         warn!("Unstable options are enabled. These options should not be used in production!");
     }
 
+    if args
+        .unstable_options
+        .contains(&UnstableOptions::OfflineMode)
+    {
+        warn!("Running in offline mode. No communication to MangaDex will be made!");
+    }
+
+    if args.unstable_options.contains(&UnstableOptions::DisableTls) {
+        warn!("Serving insecure traffic! You better be running this for development only.");
+    }
+
     if args.override_upstream.is_some()
         && !args
             .unstable_options

src/metrics.rs

@@ -0,0 +1,45 @@
+use once_cell::sync::Lazy;
+use prometheus::{register_int_counter, IntCounter};
+
+pub static CACHE_HIT_COUNTER: Lazy<IntCounter> =
+    Lazy::new(|| register_int_counter!("cache_hit", "The number of cache hits").unwrap());
+
+pub static CACHE_MISS_COUNTER: Lazy<IntCounter> =
+    Lazy::new(|| register_int_counter!("cache_miss", "The number of cache misses").unwrap());
+
+pub static REQUESTS_TOTAL_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
+    register_int_counter!("requests_total", "The total number of requests served.").unwrap()
+});
+
+pub static REQUESTS_DATA_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
+    register_int_counter!(
+        "requests_data",
+        "The number of requests served from the /data endpoint."
+    )
+    .unwrap()
+});
+
+pub static REQUESTS_DATA_SAVER_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
+    register_int_counter!(
+        "requests_data_saver",
+        "The number of requests served from the /data-saver endpoint."
+    )
+    .unwrap()
+});
+
+pub static REQUESTS_OTHER_COUNTER: Lazy<IntCounter> = Lazy::new(|| {
+    register_int_counter!(
+        "requests_other",
+        "The total number of request not served by primary endpoints."
+    )
+    .unwrap()
+});
+
+pub fn init() {
+    let _a = CACHE_HIT_COUNTER.get();
+    let _a = CACHE_MISS_COUNTER.get();
+    let _a = REQUESTS_TOTAL_COUNTER.get();
+    let _a = REQUESTS_DATA_COUNTER.get();
+    let _a = REQUESTS_DATA_SAVER_COUNTER.get();
+    let _a = REQUESTS_OTHER_COUNTER.get();
+}

src/routes.rs

@@ -1,5 +1,6 @@
 use std::sync::atomic::Ordering;
 
+use actix_web::error::ErrorNotFound;
 use actix_web::http::header::{
     ACCESS_CONTROL_ALLOW_ORIGIN, ACCESS_CONTROL_EXPOSE_HEADERS, CACHE_CONTROL, CONTENT_LENGTH,
     CONTENT_TYPE, LAST_MODIFIED, X_CONTENT_TYPE_OPTIONS,
@@ -13,6 +14,7 @@ use chrono::{DateTime, Utc};
 use futures::{Stream, TryStreamExt};
 use log::{debug, error, info, warn};
 use once_cell::sync::Lazy;
+use prometheus::{Encoder, TextEncoder};
 use reqwest::{Client, StatusCode};
 use serde::Deserialize;
 use sodiumoxide::crypto::box_::{open_precomputed, Nonce, PrecomputedKey, NONCEBYTES};
@@ -20,7 +22,11 @@ use thiserror::Error;
 
 use crate::cache::{Cache, CacheKey, ImageMetadata, UpstreamError};
 use crate::client_api_version;
-use crate::config::{SEND_SERVER_VERSION, VALIDATE_TOKENS};
+use crate::config::{OFFLINE_MODE, SEND_SERVER_VERSION, VALIDATE_TOKENS};
+use crate::metrics::{
+    CACHE_HIT_COUNTER, CACHE_MISS_COUNTER, REQUESTS_DATA_COUNTER, REQUESTS_DATA_SAVER_COUNTER,
+    REQUESTS_OTHER_COUNTER, REQUESTS_TOTAL_COUNTER,
+};
 use crate::state::RwLockServerState;
 
 pub const BASE64_CONFIG: base64::Config = base64::Config::new(base64::CharacterSet::UrlSafe, false);
@@ -46,7 +52,10 @@ impl Responder for ServerResponse {
     fn respond_to(self, req: &HttpRequest) -> HttpResponse {
         match self {
             Self::TokenValidationError(e) => e.respond_to(req),
-            Self::HttpResponse(resp) => resp.respond_to(req),
+            Self::HttpResponse(resp) => {
+                REQUESTS_TOTAL_COUNTER.inc();
+                resp.respond_to(req)
+            }
         }
     }
 }
@@ -58,6 +67,7 @@ async fn token_data(
     cache: Data<dyn Cache>,
     path: Path<(String, String, String)>,
 ) -> impl Responder {
+    REQUESTS_DATA_COUNTER.inc();
     let (token, chapter_hash, file_name) = path.into_inner();
     if VALIDATE_TOKENS.load(Ordering::Acquire) {
         if let Err(e) = validate_token(&state.0.read().precomputed_key, token, &chapter_hash) {
@@ -74,6 +84,7 @@ async fn token_data_saver(
     cache: Data<dyn Cache>,
     path: Path<(String, String, String)>,
 ) -> impl Responder {
+    REQUESTS_DATA_SAVER_COUNTER.inc();
     let (token, chapter_hash, file_name) = path.into_inner();
     if VALIDATE_TOKENS.load(Ordering::Acquire) {
         if let Err(e) = validate_token(&state.0.read().precomputed_key, token, &chapter_hash) {
@@ -86,12 +97,22 @@ async fn token_data_saver(
 
 #[allow(clippy::future_not_send)]
 pub async fn default(state: Data<RwLockServerState>, req: HttpRequest) -> impl Responder {
+    REQUESTS_OTHER_COUNTER.inc();
     let path = &format!(
         "{}{}",
         state.0.read().image_server,
         req.path().chars().skip(1).collect::<String>()
     );
+
+    if OFFLINE_MODE.load(Ordering::Acquire) {
+        info!("Got unknown path in offline mode, returning 404: {}", path);
+        return ServerResponse::HttpResponse(
+            ErrorNotFound("Path is not valid in offline mode").into(),
+        );
+    } else {
         info!("Got unknown path, just proxying: {}", path);
+    }
+
     let resp = match HTTP_CLIENT.get(path).send().await {
         Ok(resp) => resp,
         Err(e) => {
@@ -109,6 +130,17 @@ pub async fn default(state: Data<RwLockServerState>, req: HttpRequest) -> impl R
     ServerResponse::HttpResponse(resp_builder.body(resp.bytes().await.unwrap_or_default()))
 }
 
+#[allow(clippy::future_not_send)]
+#[get("/metrics")]
+pub async fn metrics() -> impl Responder {
+    let metric_families = prometheus::gather();
+    let mut buffer = Vec::new();
+    TextEncoder::new()
+        .encode(&metric_families, &mut buffer)
+        .unwrap();
+    String::from_utf8(buffer).unwrap()
+}
+
 #[derive(Error, Debug)]
 enum TokenValidationError {
     #[error("Failed to decode base64 token.")]
@@ -198,6 +230,7 @@ async fn fetch_image(
 
     match cache.get(&key).await {
         Some(Ok((image, metadata))) => {
+            CACHE_HIT_COUNTER.inc();
             return construct_response(image, &metadata);
         }
         Some(Err(_)) => {
@@ -206,6 +239,15 @@ async fn fetch_image(
         _ => (),
     }
 
+    CACHE_MISS_COUNTER.inc();
+
+    // If in offline mode, return early since there's nothing else we can do
+    if OFFLINE_MODE.load(Ordering::Acquire) {
+        return ServerResponse::HttpResponse(
+            ErrorNotFound("Offline mode enabled and image not in cache").into(),
+        );
+    }
+
     // It's important to not get a write lock before this request, else we're
     // holding the read lock until the await resolves.
 

src/state.rs

@@ -1,6 +1,7 @@
+use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, Ordering};
 
-use crate::config::{CliArgs, UnstableOptions, SEND_SERVER_VERSION, VALIDATE_TOKENS};
+use crate::config::{CliArgs, UnstableOptions, OFFLINE_MODE, SEND_SERVER_VERSION, VALIDATE_TOKENS};
 use crate::ping::{Request, Response, CONTROL_CENTER_PING_URL};
 use arc_swap::ArcSwap;
 use log::{error, info, warn};
@@ -9,7 +10,7 @@ use parking_lot::RwLock;
 use rustls::sign::{CertifiedKey, SigningKey};
 use rustls::Certificate;
 use rustls::{ClientHello, ResolvesServerCert};
-use sodiumoxide::crypto::box_::PrecomputedKey;
+use sodiumoxide::crypto::box_::{PrecomputedKey, PRECOMPUTEDKEYBYTES};
 use thiserror::Error;
 use url::Url;
 
@@ -144,6 +145,16 @@ impl ServerState {
             },
         }
     }
+
+    pub fn init_offline() -> Self {
+        assert!(OFFLINE_MODE.load(Ordering::Acquire));
+        Self {
+            precomputed_key: PrecomputedKey::from_slice(&[41; PRECOMPUTEDKEYBYTES]).unwrap(),
+            image_server: Url::from_file_path("/dev/null").unwrap(),
+            url: Url::from_str("http://localhost").unwrap(),
+            url_overridden: false,
+        }
+    }
 }
 
 pub struct RwLockServerState(pub RwLock<ServerState>);