Add web parsing tests; additional test media

This commit is contained in:
Edward Shen 2022-01-11 21:33:23 -08:00
parent 112b75afae
commit f1c40d64c7
Signed by: edward
GPG key ID: 19182661E818369F
7 changed files with 560 additions and 44 deletions

442
test/0000-test-patch.patch Normal file
View file

@ -0,0 +1,442 @@
From 960344b240161b36cca35c22b6a685162b0f217e Mon Sep 17 00:00:00 2001
From: William Tan <code@wtan.me>
Date: Tue, 11 Jan 2022 22:31:18 -0500
Subject: [PATCH] Update dependencies
---
Cargo.lock | 144 ++++++++++++++++++++++---------------------------
cli/Cargo.toml | 4 +-
2 files changed, 65 insertions(+), 83 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index fc97ae7..a16f56f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -28,9 +28,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.51"
+version = "1.0.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
+checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3"
[[package]]
name = "argon2"
@@ -149,9 +149,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "blake2"
-version = "0.10.0"
+version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a58bdf5134c5beae6fc382002c4d88950bad1feea20f8f7165494b6b43b049de"
+checksum = "b94ba84325db59637ffc528bbe8c7f86c02c57cff5c0e2b9b00f9a851f42f309"
dependencies = [
"digest 0.10.1",
]
@@ -176,9 +176,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.8.0"
+version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c"
+checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "byte-unit"
@@ -295,9 +295,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "3.0.0-rc.7"
+version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9468f8012246b0836c6fd11725102b0844254985f2462b6c637d50040ef49df0"
+checksum = "1957aa4a5fb388f0a0a73ce7556c5b42025b874e5cdc2c670775e346e97adec0"
dependencies = [
"atty",
"bitflags",
@@ -312,9 +312,9 @@ dependencies = [
[[package]]
name = "clap_derive"
-version = "3.0.0-rc.7"
+version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b72e1af32a4de4d21a43d26de33fe69c00e895371bd8b1523d674f011b610467"
+checksum = "517358c28fcef6607bf6f76108e02afad7e82297d132a6b846dcc1fc3efcd153"
dependencies = [
"heck",
"proc-macro-error",
@@ -392,9 +392,9 @@ dependencies = [
[[package]]
name = "fixedbitset"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e"
+checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"
[[package]]
name = "flate2"
@@ -515,9 +515,9 @@ dependencies = [
[[package]]
name = "generic-array"
-version = "0.14.4"
+version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
+checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
dependencies = [
"typenum",
"version_check",
@@ -556,9 +556,9 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.3.9"
+version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd"
+checksum = "0c9de88456263e249e241fcd211d3954e2c9b0ef7ccfc235a444eb367cae3689"
dependencies = [
"bytes",
"fnv",
@@ -606,12 +606,9 @@ dependencies = [
[[package]]
name = "heck"
-version = "0.3.3"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
+checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
[[package]]
name = "hermit-abi"
@@ -624,13 +621,13 @@ dependencies = [
[[package]]
name = "http"
-version = "0.2.5"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b"
+checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
dependencies = [
"bytes",
"fnv",
- "itoa 0.4.8",
+ "itoa 1.0.1",
]
[[package]]
@@ -706,9 +703,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "1.7.0"
+version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
+checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
dependencies = [
"autocfg",
"hashbrown",
@@ -921,9 +918,9 @@ dependencies = [
[[package]]
name = "num_cpus"
-version = "1.13.0"
+version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
dependencies = [
"hermit-abi",
"libc",
@@ -1086,18 +1083,18 @@ dependencies = [
[[package]]
name = "pin-project"
-version = "1.0.8"
+version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08"
+checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
-version = "1.0.8"
+version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389"
+checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb"
dependencies = [
"proc-macro2",
"quote",
@@ -1106,9 +1103,9 @@ dependencies = [
[[package]]
name = "pin-project-lite"
-version = "0.2.7"
+version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443"
+checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c"
[[package]]
name = "pin-utils"
@@ -1129,9 +1126,9 @@ dependencies = [
[[package]]
name = "ppv-lite86"
-version = "0.2.15"
+version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba"
+checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
[[package]]
name = "proc-macro-error"
@@ -1159,18 +1156,18 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.34"
+version = "1.0.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f84e92c0f7c9d58328b85a78557813e4bd845130db68d7184635344399423b1"
+checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
dependencies = [
"unicode-xid",
]
[[package]]
name = "quote"
-version = "1.0.10"
+version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05"
+checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d"
dependencies = [
"proc-macro2",
]
@@ -1258,15 +1255,16 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.11.7"
+version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bea77bc708afa10e59905c3d4af7c8fd43c9214251673095ff8b14345fcbc5"
+checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525"
dependencies = [
"base64",
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
+ "h2",
"http",
"http-body",
"hyper",
@@ -1343,7 +1341,7 @@ dependencies = [
"log",
"ring",
"sct",
- "webpki 0.22.0",
+ "webpki",
]
[[package]]
@@ -1382,18 +1380,18 @@ dependencies = [
[[package]]
name = "serde"
-version = "1.0.132"
+version = "1.0.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008"
+checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.132"
+version = "1.0.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276"
+checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537"
dependencies = [
"proc-macro2",
"quote",
@@ -1402,9 +1400,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.73"
+version = "1.0.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5"
+checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142"
dependencies = [
"itoa 1.0.1",
"ryu",
@@ -1453,9 +1451,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
[[package]]
name = "signal-hook"
-version = "0.3.12"
+version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c35dfd12afb7828318348b8c408383cf5071a086c1d4ab1c0f9840ec92dbb922"
+checksum = "647c97df271007dcea485bb74ffdb57f2e683f1306c854f468a0c244badabf2d"
dependencies = [
"libc",
"signal-hook-registry",
@@ -1472,9 +1470,9 @@ dependencies = [
[[package]]
name = "signal-hook-tokio"
-version = "0.3.0"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6c5d32165ff8b94e68e7b3bdecb1b082e958c22434b363482cfb89dcd6f3ff8"
+checksum = "213241f76fb1e37e27de3b6aa1b068a2c333233b59cca6634f634b80a27ecf1e"
dependencies = [
"futures-core",
"libc",
@@ -1524,9 +1522,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
-version = "1.0.82"
+version = "1.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59"
+checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7"
dependencies = [
"proc-macro2",
"quote",
@@ -1643,7 +1641,7 @@ checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b"
dependencies = [
"rustls",
"tokio",
- "webpki 0.22.0",
+ "webpki",
]
[[package]]
@@ -1794,9 +1792,9 @@ checksum = "e73fc24a5427b3b15e2b0bcad8ef61b5affb1da8ac89c8bf3f196c8692d57f02"
[[package]]
name = "tree_magic_mini"
-version = "3.0.2"
+version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a7581560dc616314f7d73e81419c783d93a92e7fc7331b3041ff57bab240ea6"
+checksum = "91adfd0607cacf6e4babdb870e9bec4037c1c4b151cfd279ccefc5e0c7feaa6d"
dependencies = [
"bytecount",
"fnv",
@@ -1815,9 +1813,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "typenum"
-version = "1.14.0"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "unicase"
@@ -1843,12 +1841,6 @@ dependencies = [
"tinyvec",
]
-[[package]]
-name = "unicode-segmentation"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
-
[[package]]
name = "unicode-xid"
version = "0.2.2"
@@ -1891,9 +1883,9 @@ checksum = "7cf7d77f457ef8dfa11e4cd5933c5ddb5dc52a94664071951219a97710f0a32b"
[[package]]
name = "version_check"
-version = "0.9.3"
+version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "want"
@@ -1989,16 +1981,6 @@ dependencies = [
"wasm-bindgen",
]
-[[package]]
-name = "webpki"
-version = "0.21.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
-dependencies = [
- "ring",
- "untrusted",
-]
-
[[package]]
name = "webpki"
version = "0.22.0"
@@ -2011,11 +1993,11 @@ dependencies = [
[[package]]
name = "webpki-roots"
-version = "0.21.1"
+version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940"
+checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449"
dependencies = [
- "webpki 0.21.4",
+ "webpki",
]
[[package]]
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 6d3ffe3..b47554d 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -13,6 +13,6 @@ omegaupload-common = "0.1"
anyhow = "1"
atty = "0.2"
-clap = { version = "3.0.0-rc.7", features = ["derive"] }
+clap = { version = "3", features = ["derive"] }
reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "blocking"] }
-rpassword = "5"
\ No newline at end of file
+rpassword = "5"
--
2.34.1

View file

@ -4,8 +4,12 @@ This folder contains mixed media that are under different licenses.
- `movie.mp4` is a 10 second snippet of Big Buck Bunny sourced at
http://bbb3d.renderfarming.net/download.html under the Attribution 3.0
Unported (CC BY 3.0) license.
- `movie.mkv` is identical to `movie.mp4` but transcoded as a `.mkv` file. It is
under the same license as `movie.mp4`.
- `image.png` has all rights reserved, with the sole exception of copying and
distribution for testing purposes for this project only.
- `image.png.gz` is under the same license as `image.png`.
- `image.webp` is under the same license as `image.png`.
- All other files are dual-licensed under the CC0 1.0 Universal License or MIT
No Attribution License, at your convenience.

BIN
test/image.png.gz Normal file

Binary file not shown.

BIN
test/image.webp Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 85 KiB

BIN
test/movie.mkv Normal file

Binary file not shown.

BIN
test/omegaupload Executable file

Binary file not shown.

View file

@ -57,7 +57,7 @@ pub fn decrypt(
open_in_place(&mut container, key, maybe_password)?;
let mime_type = tree_magic_mini::from_u8(&container);
log!("Mimetype: ", mime_type);
log!("Mime type: ", mime_type);
log!("Blob conversion started.");
let start = now();
@ -76,51 +76,121 @@ pub fn decrypt(
log!(format!("Blob conversion completed in {}ms", now() - start));
if mime_type.starts_with("text/") || mime_type == "application/mbox" {
if let Ok(string) = String::from_utf8(container) {
Ok(DecryptedData::String(Arc::new(string)))
} else {
Ok(DecryptedData::Blob(blob))
}
} else if mime_type.starts_with("image/")
// application/x-riff is WebP
|| mime_type == "application/x-riff"
{
Ok(DecryptedData::Image(blob, container.len()))
} else if mime_type.starts_with("audio/") {
Ok(DecryptedData::Audio(blob))
} else if mime_type.starts_with("video/")
// application/x-matroska is mkv
|| mime_type == "application/x-matroska"
{
Ok(DecryptedData::Video(blob))
} else if mime_type == "application/zip" {
let mut entries = vec![];
let cursor = Cursor::new(container);
if let Ok(mut zip) = zip::ZipArchive::new(cursor) {
for i in 0..zip.len() {
match zip.by_index(i) {
Ok(file) => entries.push(ArchiveMeta {
name: file.name().to_string(),
file_size: file.size(),
}),
Err(err) => match err {
zip::result::ZipError::UnsupportedArchive(s) => {
log!("Unsupported: ", s.to_string());
}
_ => {
log!(format!("Error: {}", err));
}
},
match container.content_type() {
ContentType::Text => Ok(DecryptedData::String(Arc::new(
// SAFETY: ContentType::Text is guaranteed to be valid UTF-8.
unsafe { String::from_utf8_unchecked(container) },
))),
ContentType::Image => Ok(DecryptedData::Image(blob, container.len())),
ContentType::Audio => Ok(DecryptedData::Audio(blob)),
ContentType::Video => Ok(DecryptedData::Video(blob)),
ContentType::ZipArchive => {
let mut entries = vec![];
let cursor = Cursor::new(container);
if let Ok(mut zip) = zip::ZipArchive::new(cursor) {
for i in 0..zip.len() {
match zip.by_index(i) {
Ok(file) => entries.push(ArchiveMeta {
name: file.name().to_string(),
file_size: file.size(),
}),
Err(err) => match err {
zip::result::ZipError::UnsupportedArchive(s) => {
log!("Unsupported: ", s.to_string());
}
_ => {
log!(format!("Error: {}", err));
}
},
}
}
}
}
entries.sort_by(|a, b| a.name.cmp(&b.name));
Ok(DecryptedData::Archive(blob, entries))
} else if mime_type == "application/gzip" {
Ok(DecryptedData::Archive(blob, vec![]))
} else {
Ok(DecryptedData::Blob(blob))
entries.sort_by(|a, b| a.name.cmp(&b.name));
Ok(DecryptedData::Archive(blob, entries))
}
ContentType::GzipArchive => Ok(DecryptedData::Archive(blob, vec![])),
ContentType::Unknown => Ok(DecryptedData::Blob(blob)),
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
enum ContentType {
Text,
Image,
Audio,
Video,
ZipArchive,
GzipArchive,
Unknown,
}
trait ContentTypeExt {
fn mime_type(&self) -> &str;
fn content_type(&self) -> ContentType;
}
impl<T: AsRef<[u8]>> ContentTypeExt for T {
fn mime_type(&self) -> &str {
tree_magic_mini::from_u8(self.as_ref())
}
fn content_type(&self) -> ContentType {
let mime_type = self.mime_type();
if mime_type.starts_with("text/") || mime_type == "application/mbox" {
if std::str::from_utf8(self.as_ref()).is_ok() {
ContentType::Text
} else {
ContentType::Unknown
}
} else if mime_type.starts_with("image/")
// application/x-riff is WebP
|| mime_type == "application/x-riff"
{
ContentType::Image
} else if mime_type.starts_with("audio/") {
ContentType::Audio
} else if mime_type.starts_with("video/")
// application/x-matroska is mkv
|| mime_type == "application/x-matroska"
{
ContentType::Video
} else if mime_type == "application/zip" {
ContentType::ZipArchive
} else if mime_type == "application/gzip" {
ContentType::GzipArchive
} else {
ContentType::Unknown
}
}
}
#[cfg(test)]
mod content_type {
use super::*;
macro_rules! test_content_type {
($($name:ident, $path:literal, $type:expr),*) => {
$(
#[test]
fn $name() {
let data = include_bytes!(concat!("../../test/", $path));
assert_eq!(data.content_type(), $type);
}
)*
};
}
test_content_type!(license_is_text, "LICENSE.md", ContentType::Text);
test_content_type!(code_is_text, "code.rs", ContentType::Text);
test_content_type!(patch_is_text, "0000-test-patch.patch", ContentType::Text);
test_content_type!(png_is_image, "image.png", ContentType::Image);
test_content_type!(webp_is_image, "image.webp", ContentType::Image);
test_content_type!(svg_is_image, "image.svg", ContentType::Image);
test_content_type!(mp3_is_audio, "music.mp3", ContentType::Audio);
test_content_type!(mp4_is_video, "movie.mp4", ContentType::Video);
test_content_type!(mkv_is_video, "movie.mkv", ContentType::Video);
test_content_type!(zip_is_zip, "archive.zip", ContentType::ZipArchive);
test_content_type!(gzip_is_gzip, "image.png.gz", ContentType::GzipArchive);
test_content_type!(binary_is_unknown, "omegaupload", ContentType::Unknown);
}