2021-04-22 22:09:48 +00:00
|
|
|
#![warn(clippy::pedantic, clippy::nursery)]
|
2021-03-23 00:00:21 +00:00
|
|
|
// We're end users, so these is ok
|
2021-04-18 03:19:27 +00:00
|
|
|
#![allow(clippy::module_name_repetitions)]
|
2021-03-18 01:45:16 +00:00
|
|
|
|
2021-07-09 23:48:25 +00:00
|
|
|
use std::env::VarError;
|
2021-05-12 01:01:01 +00:00
|
|
|
use std::error::Error;
|
|
|
|
use std::fmt::Display;
|
2021-07-15 06:14:04 +00:00
|
|
|
use std::net::SocketAddr;
|
2021-07-09 21:18:43 +00:00
|
|
|
use std::num::ParseIntError;
|
2021-07-15 06:14:04 +00:00
|
|
|
use std::str::FromStr;
|
2021-04-23 22:03:53 +00:00
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
2021-03-22 21:47:56 +00:00
|
|
|
use std::sync::Arc;
|
2021-03-18 01:45:16 +00:00
|
|
|
use std::time::Duration;
|
|
|
|
|
2021-07-15 06:14:04 +00:00
|
|
|
use actix_web::dev::Service;
|
2021-03-22 21:47:56 +00:00
|
|
|
use actix_web::rt::{spawn, time, System};
|
2021-03-23 03:04:54 +00:00
|
|
|
use actix_web::web::{self, Data};
|
2021-06-24 14:39:12 +00:00
|
|
|
use actix_web::{App, HttpResponse, HttpServer};
|
2021-04-23 21:22:29 +00:00
|
|
|
use cache::{Cache, DiskCache};
|
2021-07-14 00:38:01 +00:00
|
|
|
use chacha20::Key;
|
2021-07-09 21:18:43 +00:00
|
|
|
use config::Config;
|
2021-07-15 06:14:04 +00:00
|
|
|
use maxminddb::geoip2;
|
2021-04-19 04:16:13 +00:00
|
|
|
use parking_lot::RwLock;
|
2021-07-17 16:52:02 +00:00
|
|
|
use redis::Client as RedisClient;
|
2021-12-02 06:08:40 +00:00
|
|
|
|
|
|
|
use rustls::server::NoClientAuth;
|
|
|
|
use rustls::ServerConfig;
|
2021-07-14 00:38:01 +00:00
|
|
|
use sodiumoxide::crypto::stream::xchacha20::gen_key;
|
2021-03-22 21:47:56 +00:00
|
|
|
use state::{RwLockServerState, ServerState};
|
|
|
|
use stop::send_stop;
|
2021-03-18 01:45:16 +00:00
|
|
|
use thiserror::Error;
|
2021-07-13 03:23:51 +00:00
|
|
|
use tracing::{debug, error, info, warn};
|
2021-03-18 01:45:16 +00:00
|
|
|
|
2021-05-20 02:42:44 +00:00
|
|
|
use crate::cache::mem::{Lfu, Lru};
|
|
|
|
use crate::cache::{MemoryCache, ENCRYPTION_KEY};
|
2021-07-09 21:18:43 +00:00
|
|
|
use crate::config::{CacheType, UnstableOptions, OFFLINE_MODE};
|
2021-07-15 06:14:04 +00:00
|
|
|
use crate::metrics::{record_country_visit, GEOIP_DATABASE};
|
2021-04-24 04:56:58 +00:00
|
|
|
use crate::state::DynamicServerCert;
|
2021-04-23 04:11:30 +00:00
|
|
|
|
2021-03-22 21:47:56 +00:00
|
|
|
mod cache;
|
2021-07-10 22:53:28 +00:00
|
|
|
mod client;
|
2021-03-26 01:06:54 +00:00
|
|
|
mod config;
|
2021-05-23 02:10:03 +00:00
|
|
|
mod metrics;
|
2021-03-18 01:45:16 +00:00
|
|
|
mod ping;
|
|
|
|
mod routes;
|
2021-03-22 21:47:56 +00:00
|
|
|
mod state;
|
2021-03-18 01:45:16 +00:00
|
|
|
mod stop;
|
2021-06-16 19:31:03 +00:00
|
|
|
mod units;
|
2021-03-18 01:45:16 +00:00
|
|
|
|
2021-07-09 21:18:43 +00:00
|
|
|
const CLIENT_API_VERSION: usize = 31;
|
2021-03-26 02:58:07 +00:00
|
|
|
|
2021-03-18 01:45:16 +00:00
|
|
|
#[derive(Error, Debug)]
|
|
|
|
enum ServerError {
|
|
|
|
#[error("There was a failure parsing config")]
|
|
|
|
Config(#[from] VarError),
|
|
|
|
#[error("Failed to parse an int")]
|
|
|
|
ParseInt(#[from] ParseIntError),
|
|
|
|
}
|
|
|
|
|
|
|
|
#[actix_web::main]
|
2021-04-25 16:55:31 +00:00
|
|
|
async fn main() -> Result<(), Box<dyn Error>> {
|
2021-05-12 01:01:01 +00:00
|
|
|
sodiumoxide::init().expect("Failed to initialize crypto");
|
2021-03-22 21:47:56 +00:00
|
|
|
// It's ok to fail early here, it would imply we have a invalid config.
|
2021-03-18 01:45:16 +00:00
|
|
|
dotenv::dotenv().ok();
|
2021-04-18 02:12:02 +00:00
|
|
|
|
2021-05-23 03:06:05 +00:00
|
|
|
//
|
|
|
|
// Config loading
|
|
|
|
//
|
|
|
|
|
2021-07-09 23:48:25 +00:00
|
|
|
let config = match config::load_config() {
|
|
|
|
Ok(c) => c,
|
|
|
|
Err(e) => {
|
|
|
|
eprintln!("{}", e);
|
|
|
|
return Err(Box::new(e) as Box<_>);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2021-07-09 21:18:43 +00:00
|
|
|
let memory_quota = config.memory_quota;
|
|
|
|
let disk_quota = config.disk_quota;
|
|
|
|
let cache_type = config.cache_type;
|
|
|
|
let cache_path = config.cache_path.clone();
|
|
|
|
let disable_tls = config
|
2021-05-23 03:06:05 +00:00
|
|
|
.unstable_options
|
|
|
|
.contains(&UnstableOptions::DisableTls);
|
2021-07-09 21:18:43 +00:00
|
|
|
let bind_address = config.bind_address;
|
2021-07-17 16:52:02 +00:00
|
|
|
let redis_url = config.redis_url.clone();
|
2021-05-23 03:06:05 +00:00
|
|
|
|
|
|
|
//
|
|
|
|
// Logging and warnings
|
|
|
|
//
|
2021-03-26 01:06:54 +00:00
|
|
|
|
2021-07-13 03:23:51 +00:00
|
|
|
tracing_subscriber::fmt()
|
|
|
|
.with_max_level(config.log_level)
|
|
|
|
.init();
|
2021-03-26 01:06:54 +00:00
|
|
|
|
2021-07-09 21:18:43 +00:00
|
|
|
if let Err(e) = print_preamble_and_warnings(&config) {
|
2021-04-25 16:55:31 +00:00
|
|
|
error!("{}", e);
|
|
|
|
return Err(e);
|
|
|
|
}
|
2021-04-19 03:06:18 +00:00
|
|
|
|
2021-07-09 23:51:48 +00:00
|
|
|
debug!("{:?}", &config);
|
|
|
|
|
2021-07-09 23:48:25 +00:00
|
|
|
let client_secret = config.client_secret.clone();
|
|
|
|
let client_secret_1 = config.client_secret.clone();
|
2021-03-26 01:06:54 +00:00
|
|
|
|
2021-07-09 21:18:43 +00:00
|
|
|
if config.ephemeral_disk_encryption {
|
2021-05-20 01:42:55 +00:00
|
|
|
info!("Running with at-rest encryption!");
|
2021-07-14 00:38:01 +00:00
|
|
|
ENCRYPTION_KEY
|
|
|
|
.set(*Key::from_slice(gen_key().as_ref()))
|
|
|
|
.unwrap();
|
2021-05-20 01:42:55 +00:00
|
|
|
}
|
|
|
|
|
2021-07-09 23:14:53 +00:00
|
|
|
if config.enable_metrics {
|
|
|
|
metrics::init();
|
|
|
|
}
|
2021-05-23 03:06:05 +00:00
|
|
|
|
2021-07-15 06:14:04 +00:00
|
|
|
if let Some(key) = config.geoip_license_key.clone() {
|
|
|
|
if let Err(e) = metrics::load_geo_ip_data(key).await {
|
|
|
|
error!("Failed to initialize geo ip db: {}", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-23 02:10:03 +00:00
|
|
|
// HTTP Server init
|
|
|
|
|
2021-07-22 17:37:32 +00:00
|
|
|
// Try bind to provided port first
|
|
|
|
let port_reservation = std::net::TcpListener::bind(bind_address);
|
|
|
|
if let Err(e) = port_reservation {
|
|
|
|
error!("Failed to bind to port!");
|
|
|
|
return Err(e.into());
|
|
|
|
};
|
|
|
|
|
2021-05-23 03:06:05 +00:00
|
|
|
let server = if OFFLINE_MODE.load(Ordering::Acquire) {
|
|
|
|
ServerState::init_offline()
|
|
|
|
} else {
|
2021-07-09 21:18:43 +00:00
|
|
|
ServerState::init(&client_secret, &config).await?
|
2021-05-23 03:06:05 +00:00
|
|
|
};
|
2021-03-26 02:58:07 +00:00
|
|
|
let data_0 = Arc::new(RwLockServerState(RwLock::new(server)));
|
|
|
|
let data_1 = Arc::clone(&data_0);
|
|
|
|
|
|
|
|
//
|
|
|
|
// At this point, the server is ready to start, and starts the necessary
|
|
|
|
// threads.
|
|
|
|
//
|
2021-03-22 21:47:56 +00:00
|
|
|
|
|
|
|
// Set ctrl+c to send a stop message
|
|
|
|
let running = Arc::new(AtomicBool::new(true));
|
2021-04-19 03:06:18 +00:00
|
|
|
let running_1 = running.clone();
|
|
|
|
let system = System::current();
|
2021-03-22 21:47:56 +00:00
|
|
|
ctrlc::set_handler(move || {
|
2021-04-19 03:06:18 +00:00
|
|
|
let system = &system;
|
2021-03-22 21:47:56 +00:00
|
|
|
let client_secret = client_secret.clone();
|
2021-04-19 03:06:18 +00:00
|
|
|
let running_2 = Arc::clone(&running_1);
|
2021-05-23 03:06:05 +00:00
|
|
|
if !OFFLINE_MODE.load(Ordering::Acquire) {
|
|
|
|
System::new().block_on(async move {
|
|
|
|
if running_2.load(Ordering::SeqCst) {
|
|
|
|
send_stop(&client_secret).await;
|
|
|
|
} else {
|
|
|
|
warn!("Got second Ctrl-C, forcefully exiting");
|
2021-07-11 17:19:37 +00:00
|
|
|
system.stop();
|
2021-05-23 03:06:05 +00:00
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
2021-04-19 03:06:18 +00:00
|
|
|
running_1.store(false, Ordering::SeqCst);
|
2021-03-22 21:47:56 +00:00
|
|
|
})
|
|
|
|
.expect("Error setting Ctrl-C handler");
|
|
|
|
|
2021-03-26 02:58:07 +00:00
|
|
|
// Spawn ping task
|
2021-05-23 03:06:05 +00:00
|
|
|
if !OFFLINE_MODE.load(Ordering::Acquire) {
|
|
|
|
spawn(async move {
|
|
|
|
let mut interval = time::interval(Duration::from_secs(90));
|
|
|
|
let mut data = Arc::clone(&data_0);
|
|
|
|
loop {
|
|
|
|
interval.tick().await;
|
|
|
|
debug!("Sending ping!");
|
2021-07-09 21:18:43 +00:00
|
|
|
ping::update_server_state(&client_secret_1, &config, &mut data).await;
|
2021-05-23 03:06:05 +00:00
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
2021-03-18 01:45:16 +00:00
|
|
|
|
2021-07-09 21:18:43 +00:00
|
|
|
let memory_max_size = memory_quota.into();
|
|
|
|
let cache = DiskCache::new(disk_quota.into(), cache_path.clone()).await;
|
|
|
|
let cache: Arc<dyn Cache> = match cache_type {
|
|
|
|
CacheType::OnDisk => cache,
|
2021-07-16 05:13:31 +00:00
|
|
|
CacheType::Lru => MemoryCache::<Lfu, _>::new(cache, memory_max_size),
|
|
|
|
CacheType::Lfu => MemoryCache::<Lru, _>::new(cache, memory_max_size),
|
2021-07-17 16:52:02 +00:00
|
|
|
CacheType::Redis => {
|
|
|
|
let url = redis_url.unwrap_or_else(|| {
|
|
|
|
url::Url::parse("redis://127.0.0.1/").expect("default redis url to be parsable")
|
|
|
|
});
|
|
|
|
info!("Trying to connect to redis instance at {}", url);
|
|
|
|
let mem_cache = RedisClient::open(url)?;
|
|
|
|
Arc::new(MemoryCache::new_with_cache(cache, mem_cache))
|
|
|
|
}
|
2021-04-23 04:11:30 +00:00
|
|
|
};
|
|
|
|
|
2021-04-22 17:11:08 +00:00
|
|
|
let cache_0 = Arc::clone(&cache);
|
2021-04-18 21:38:33 +00:00
|
|
|
|
2021-03-26 02:58:07 +00:00
|
|
|
// Start HTTPS server
|
2021-05-23 03:06:05 +00:00
|
|
|
let server = HttpServer::new(move || {
|
2021-03-18 01:45:16 +00:00
|
|
|
App::new()
|
2021-07-15 06:14:04 +00:00
|
|
|
.wrap_fn(|req, srv| {
|
|
|
|
if let Some(reader) = GEOIP_DATABASE.get() {
|
|
|
|
let maybe_country = req
|
|
|
|
.connection_info()
|
|
|
|
.realip_remote_addr()
|
|
|
|
.map(SocketAddr::from_str)
|
|
|
|
.and_then(Result::ok)
|
|
|
|
.as_ref()
|
|
|
|
.map(SocketAddr::ip)
|
|
|
|
.map(|ip| reader.lookup::<geoip2::Country>(ip))
|
|
|
|
.and_then(Result::ok);
|
|
|
|
|
|
|
|
record_country_visit(maybe_country);
|
|
|
|
}
|
|
|
|
|
|
|
|
srv.call(req)
|
|
|
|
})
|
2021-05-27 21:05:50 +00:00
|
|
|
.service(routes::index)
|
2021-03-18 01:45:16 +00:00
|
|
|
.service(routes::token_data)
|
2021-03-22 21:47:56 +00:00
|
|
|
.service(routes::token_data_saver)
|
2021-05-23 03:06:05 +00:00
|
|
|
.service(routes::metrics)
|
2021-06-24 14:39:12 +00:00
|
|
|
.route(
|
|
|
|
"/data/{tail:.*}",
|
|
|
|
web::get().to(HttpResponse::UnavailableForLegalReasons),
|
|
|
|
)
|
|
|
|
.route(
|
|
|
|
"/data-saver/{tail:.*}",
|
|
|
|
web::get().to(HttpResponse::UnavailableForLegalReasons),
|
|
|
|
)
|
2021-03-22 21:47:56 +00:00
|
|
|
.route("{tail:.*}", web::get().to(routes::default))
|
2021-03-18 01:45:16 +00:00
|
|
|
.app_data(Data::from(Arc::clone(&data_1)))
|
2021-04-22 17:11:08 +00:00
|
|
|
.app_data(Data::from(Arc::clone(&cache_0)))
|
2021-03-18 01:45:16 +00:00
|
|
|
})
|
2021-05-23 03:06:05 +00:00
|
|
|
.shutdown_timeout(60);
|
|
|
|
|
2021-07-22 17:37:32 +00:00
|
|
|
// drop port reservation, might have a TOCTOU but it's not a big deal; this
|
|
|
|
// is just a best effort.
|
|
|
|
std::mem::drop(port_reservation);
|
|
|
|
|
2021-05-23 03:06:05 +00:00
|
|
|
if disable_tls {
|
2021-07-09 21:18:43 +00:00
|
|
|
server.bind(bind_address)?.run().await?;
|
2021-05-23 03:06:05 +00:00
|
|
|
} else {
|
2021-07-09 21:18:43 +00:00
|
|
|
// Rustls only supports TLS 1.2 and 1.3.
|
2021-12-02 06:08:40 +00:00
|
|
|
let tls_config = ServerConfig::builder()
|
|
|
|
.with_safe_defaults()
|
|
|
|
.with_client_cert_verifier(NoClientAuth::new())
|
|
|
|
.with_cert_resolver(Arc::new(DynamicServerCert));
|
2021-07-09 21:18:43 +00:00
|
|
|
|
|
|
|
server.bind_rustls(bind_address, tls_config)?.run().await?;
|
2021-05-23 03:06:05 +00:00
|
|
|
}
|
2021-03-22 21:47:56 +00:00
|
|
|
|
|
|
|
// Waiting for us to finish sending stop message
|
|
|
|
while running.load(Ordering::SeqCst) {
|
2021-07-15 16:29:55 +00:00
|
|
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
2021-03-22 21:47:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
2021-03-18 01:45:16 +00:00
|
|
|
}
|
2021-04-19 03:06:18 +00:00
|
|
|
|
2021-04-25 16:55:31 +00:00
|
|
|
#[derive(Debug)]
|
|
|
|
enum InvalidCombination {
|
|
|
|
MissingUnstableOption(&'static str, UnstableOptions),
|
|
|
|
}
|
|
|
|
|
2021-07-20 20:47:04 +00:00
|
|
|
#[cfg(not(tarpaulin_include))]
|
2021-04-25 16:55:31 +00:00
|
|
|
impl Display for InvalidCombination {
|
|
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
|
|
match self {
|
|
|
|
InvalidCombination::MissingUnstableOption(opt, arg) => {
|
|
|
|
write!(
|
|
|
|
f,
|
|
|
|
"The option '{}' requires the unstable option '-Z {}'",
|
|
|
|
opt, arg
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Error for InvalidCombination {}
|
|
|
|
|
2021-07-20 20:47:04 +00:00
|
|
|
#[cfg(not(tarpaulin_include))]
|
2022-03-26 23:28:58 +00:00
|
|
|
#[allow(clippy::cognitive_complexity)]
|
2021-07-09 21:18:43 +00:00
|
|
|
fn print_preamble_and_warnings(args: &Config) -> Result<(), Box<dyn Error>> {
|
2021-07-15 06:45:05 +00:00
|
|
|
let build_string = option_env!("VERGEN_GIT_SHA_SHORT")
|
|
|
|
.map(|git_sha| format!(" ({})", git_sha))
|
|
|
|
.unwrap_or_default();
|
|
|
|
|
|
|
|
println!(
|
|
|
|
concat!(
|
|
|
|
env!("CARGO_PKG_NAME"),
|
|
|
|
" ",
|
|
|
|
env!("CARGO_PKG_VERSION"),
|
|
|
|
"{} Copyright (C) 2021 ",
|
|
|
|
env!("CARGO_PKG_AUTHORS"),
|
|
|
|
"\n\n",
|
|
|
|
env!("CARGO_PKG_NAME"),
|
|
|
|
" is free software: you can redistribute it and/or modify\n\
|
|
|
|
it under the terms of the GNU General Public License as published by\n\
|
|
|
|
the Free Software Foundation, either version 3 of the License, or\n\
|
|
|
|
(at your option) any later version.\n\n",
|
|
|
|
env!("CARGO_PKG_NAME"),
|
|
|
|
" is distributed in the hope that it will be useful,\n\
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
|
|
|
|
GNU General Public License for more details.\n\n\
|
|
|
|
You should have received a copy of the GNU General Public License\n\
|
|
|
|
along with ",
|
|
|
|
env!("CARGO_PKG_NAME"),
|
|
|
|
". If not, see <https://www.gnu.org/licenses/>.\n"
|
|
|
|
),
|
|
|
|
build_string
|
|
|
|
);
|
2021-04-25 16:55:31 +00:00
|
|
|
|
|
|
|
if !args.unstable_options.is_empty() {
|
|
|
|
warn!("Unstable options are enabled. These options should not be used in production!");
|
|
|
|
}
|
|
|
|
|
2021-05-23 03:06:05 +00:00
|
|
|
if args
|
|
|
|
.unstable_options
|
|
|
|
.contains(&UnstableOptions::OfflineMode)
|
|
|
|
{
|
|
|
|
warn!("Running in offline mode. No communication to MangaDex will be made!");
|
|
|
|
}
|
|
|
|
|
|
|
|
if args.unstable_options.contains(&UnstableOptions::DisableTls) {
|
|
|
|
warn!("Serving insecure traffic! You better be running this for development only.");
|
|
|
|
}
|
|
|
|
|
2021-07-15 16:29:55 +00:00
|
|
|
if args
|
|
|
|
.unstable_options
|
|
|
|
.contains(&UnstableOptions::DisableCertValidation)
|
|
|
|
{
|
|
|
|
error!("Cert validation disabled! You REALLY only better be debugging.");
|
|
|
|
}
|
|
|
|
|
2021-04-25 16:55:31 +00:00
|
|
|
if args.override_upstream.is_some()
|
|
|
|
&& !args
|
|
|
|
.unstable_options
|
|
|
|
.contains(&UnstableOptions::OverrideUpstream)
|
|
|
|
{
|
|
|
|
Err(Box::new(InvalidCombination::MissingUnstableOption(
|
|
|
|
"override-upstream",
|
|
|
|
UnstableOptions::OverrideUpstream,
|
|
|
|
)))
|
|
|
|
} else {
|
|
|
|
Ok(())
|
|
|
|
}
|
2021-04-19 03:06:18 +00:00
|
|
|
}
|