diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1e12134ea8cf8e8d87d8cf7f8f6c69e3bb4c9270..b4063cb5edf1072e714cef4a67f67ab1e8b02dd7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -80,18 +80,14 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- - name: Install nightly toolchain
- run: rustup toolchain install nightly --component llvm-tools-preview
- - name: Install cargo-llvm-cov
- uses: taiki-e/install-action@cargo-llvm-cov
- - name: Generate code coverage
- run: cargo llvm-cov --features=all,integration_tests --lcov --output-path lcov.info -- --test-threads 1
+ - name: Install stable toolchain
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
+ - name: Run tests
+ run: cargo test --features=all,integration_tests -- --test-threads 1
env:
TEST_REDIS_URL: redis://localhost:${{ job.services.redis.ports['6379'] }}/0
TEST_DB_URL: postgresql://rustus:rustus@localhost:${{ job.services.pg.ports['5432'] }}/rustus
TEST_AMQP_URL: amqp://guest:guest@localhost:${{ job.services.rabbit.ports['5672'] }}
- - name: Coveralls
- uses: coverallsapp/github-action@master
- with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
- path-to-lcov: lcov.info
diff --git a/Cargo.lock b/Cargo.lock
index d345c908eaf3da88af2a184ec1c1434835f3d4f7..6e31f9faf413c8aae4f31879a95b3ef4eee13209 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1456,6 +1456,15 @@ dependencies = [
"opaque-debug 0.3.0",
]
+[[package]]
+name = "md-5"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582"
+dependencies = [
+ "digest 0.10.3",
+]
+
[[package]]
name = "memchr"
version = "2.4.1"
@@ -2192,7 +2201,7 @@ dependencies = [
"itoa 0.4.8",
"percent-encoding",
"pin-project-lite",
- "sha1",
+ "sha1 0.6.1",
"tokio",
"tokio-util 0.6.9",
"url",
@@ -2385,12 +2394,14 @@ dependencies = [
"bytes",
"chrono",
"derive_more",
+ "digest 0.10.3",
"fern",
"futures",
"httptest",
"lapin",
"lazy_static",
"log",
+ "md-5 0.10.1",
"mobc-lapin",
"mobc-redis",
"openssl",
@@ -2399,6 +2410,8 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
+ "sha1 0.10.1",
+ "sha2 0.10.2",
"strfmt",
"structopt",
"strum",
@@ -2590,6 +2603,17 @@ dependencies = [
"sha1_smol",
]
+[[package]]
+name = "sha1"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c77f4e7f65455545c2153c1253d25056825e77ee2533f0e41deb65a93a34852f"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest 0.10.3",
+]
+
[[package]]
name = "sha1_smol"
version = "1.0.0"
@@ -2609,6 +2633,17 @@ dependencies = [
"opaque-debug 0.3.0",
]
+[[package]]
+name = "sha2"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest 0.10.3",
+]
+
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
@@ -2711,7 +2746,7 @@ dependencies = [
"libc",
"libsqlite3-sys",
"log",
- "md-5",
+ "md-5 0.9.1",
"memchr",
"num-bigint",
"once_cell",
@@ -2725,7 +2760,7 @@ dependencies = [
"serde",
"serde_json",
"sha-1 0.9.8",
- "sha2",
+ "sha2 0.9.9",
"smallvec",
"sqlformat",
"sqlx-rt",
@@ -2799,7 +2834,7 @@ dependencies = [
"serde",
"serde_derive",
"serde_json",
- "sha1",
+ "sha1 0.6.1",
"syn",
]
diff --git a/Cargo.toml b/Cargo.toml
index cc0d737506ef9fa5702888304da29c1ba66c97f9..33db9d71e422cb2353b8c42669452cc7e03d4d3c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,6 +31,24 @@ thiserror = "^1.0"
url = "^2.2.2"
bytes = "^1.1.0"
+[dependencies.digest]
+version = "0.10.3"
+optional = true
+
+[dependencies.sha1]
+version = "^0.10.1"
+features = ["compress"]
+optional = true
+
+[dependencies.sha2]
+version = "^0.10.1"
+features = ["compress"]
+optional = true
+
+[dependencies.md-5]
+version = "^0.10.1"
+optional = true
+
[dependencies.futures]
version = "^0.3.21"
@@ -108,12 +126,13 @@ features = ["v4"]
version = "^1.0.0-alpha.1"
[features]
-all = ["redis_info_storage", "db_info_storage", "http_notifier", "amqp_notifier"]
+all = ["redis_info_storage", "db_info_storage", "http_notifier", "amqp_notifier", "hashers"]
amqp_notifier = ["lapin", "tokio-amqp", "mobc-lapin"]
db_info_storage = ["rbatis", "rbson"]
default = []
http_notifier = ["reqwest"]
redis_info_storage = ["mobc-redis"]
+hashers = ["md-5", "sha1", "sha2", "digest"]
### For testing
test_redis = []
diff --git a/src/config.rs b/src/config.rs
index 50b8b79379c6a4183f91c1c2f82f816dc52e342b..ae38eaa1b8800262d71fcf95d29069057812e2bd 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -189,7 +189,7 @@ pub struct RustusConf {
/// Enabled extensions for TUS protocol.
#[structopt(
long,
- default_value = "getting,creation,termination,creation-with-upload,creation-defer-length,concatenation",
+ default_value = "getting,creation,termination,creation-with-upload,creation-defer-length,concatenation,checksum",
env = "RUSTUS_TUS_EXTENSIONS",
use_delimiter = true
)]
diff --git a/src/errors.rs b/src/errors.rs
index c2a4f49ae14e4e504cfbb4b7abb869c81572257d..13ad2ae57a614f7cbae16168fb21eef0185fea64 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -59,6 +59,12 @@ pub enum RustusError {
StdError(#[from] std::io::Error),
#[error("Can't spawn task: {0}")]
TokioSpawnError(#[from] tokio::task::JoinError),
+ #[error("Unknown hashsum algorithm")]
+ UnknownHashAlgorithm,
+ #[error("Wrong checksum")]
+ WrongChecksum,
+ #[error("The header value is incorrect")]
+ WrongHeaderValue,
}
/// This conversion allows us to use `RustusError` in the `main` function.
@@ -83,9 +89,12 @@ impl ResponseError for RustusError {
match self {
RustusError::FileNotFound => StatusCode::NOT_FOUND,
RustusError::WrongOffset => StatusCode::CONFLICT,
- RustusError::FrozenFile | RustusError::SizeAlreadyKnown | RustusError::HookError(_) => {
- StatusCode::BAD_REQUEST
- }
+ RustusError::FrozenFile
+ | RustusError::SizeAlreadyKnown
+ | RustusError::HookError(_)
+ | RustusError::UnknownHashAlgorithm
+ | RustusError::WrongHeaderValue => StatusCode::BAD_REQUEST,
+ RustusError::WrongChecksum => StatusCode::EXPECTATION_FAILED,
_ => StatusCode::INTERNAL_SERVER_ERROR,
}
}
diff --git a/src/protocol/core/server_info.rs b/src/protocol/core/server_info.rs
index 3648e84ea46aadf08c003be733c86aaf1b627635..41d1e2f9a5cf424993158a4178b213bbaed3801a 100644
--- a/src/protocol/core/server_info.rs
+++ b/src/protocol/core/server_info.rs
@@ -1,4 +1,6 @@
-use actix_web::{web, HttpResponse};
+#[cfg(feature = "hashers")]
+use crate::protocol::extensions::Extensions;
+use actix_web::{http::StatusCode, web, HttpResponse, HttpResponseBuilder};
use crate::State;
@@ -12,9 +14,13 @@ pub async fn server_info(state: web::Data<State>) -> HttpResponse {
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join(",");
- HttpResponse::Ok()
- .insert_header(("Tus-Extension", ext_str.as_str()))
- .finish()
+ let mut response_builder = HttpResponseBuilder::new(StatusCode::OK);
+ response_builder.insert_header(("Tus-Extension", ext_str.as_str()));
+ #[cfg(feature = "hashers")]
+ if state.config.tus_extensions.contains(&Extensions::Checksum) {
+ response_builder.insert_header(("Tus-Checksum-Algorithm", "md5,sha1,sha256,sha512"));
+ }
+ response_builder.finish()
}
#[cfg(test)]
diff --git a/src/protocol/core/write_bytes.rs b/src/protocol/core/write_bytes.rs
index 49418abfcca6a3b82f1bdb1faddba87b9b7f16a1..fafc64e59c9da415eb623f50cb8fc5178046dcf2 100644
--- a/src/protocol/core/write_bytes.rs
+++ b/src/protocol/core/write_bytes.rs
@@ -1,5 +1,7 @@
use actix_web::{web, web::Bytes, HttpRequest, HttpResponse};
+#[cfg(feature = "hashers")]
+use crate::utils::hashes::verify_chunk_checksum;
use crate::{
errors::RustusError,
notifiers::Hook,
@@ -29,6 +31,20 @@ pub async fn write_bytes(
return Err(RustusError::FileNotFound);
}
+ #[cfg(feature = "hashers")]
+ if state.config.tus_extensions.contains(&Extensions::Checksum) {
+ if let Some(header) = request.headers().get("Upload-Checksum").cloned() {
+ let cloned_bytes = bytes.clone();
+ if !tokio::task::spawn_blocking(move || {
+ verify_chunk_checksum(&header, cloned_bytes.as_ref())
+ })
+ .await??
+ {
+ return Err(RustusError::WrongChecksum);
+ }
+ }
+ }
+
// New upload length.
// Parses header `Upload-Length` only if the creation-defer-length extension is enabled.
let updated_len = if state
@@ -151,6 +167,7 @@ mod tests {
let request = TestRequest::patch()
.uri(state.config.file_url(file.id.as_str()).as_str())
.insert_header(("Content-Type", "application/offset+octet-stream"))
+ .insert_header(("Upload-Checksum", "md5 xIwpFX4rNYzBRAJ/Pi2MtA=="))
.insert_header(("Upload-Offset", file.offset))
.set_payload(test_data)
.to_request();
@@ -418,4 +435,27 @@ mod tests {
let resp = call_service(&mut rustus, request).await;
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
+
+ #[actix_rt::test]
+ /// Tests checksum validation.
+ async fn wrong_checksum() {
+ let state = State::test_new().await;
+ let mut rustus = init_service(
+ App::new().configure(rustus_service(web::Data::new(state.test_clone().await))),
+ )
+ .await;
+ let mut file = state.create_test_file().await;
+ file.offset = 0;
+ file.length = Some(10);
+ state.info_storage.set_info(&file, false).await.unwrap();
+ let request = TestRequest::patch()
+ .uri(state.config.file_url(file.id.as_str()).as_str())
+ .insert_header(("Upload-Offset", "0"))
+ .insert_header(("Upload-Checksum", "md5 K9opmNmw7hl9oUKgRH9nJQ=="))
+ .insert_header(("Content-Type", "application/offset+octet-stream"))
+ .set_payload("memes")
+ .to_request();
+ let resp = call_service(&mut rustus, request).await;
+ assert_eq!(resp.status(), StatusCode::EXPECTATION_FAILED);
+ }
}
diff --git a/src/protocol/extensions.rs b/src/protocol/extensions.rs
index 4eba16223bc6be6c6971362922d5e93cf10ac54d..9847523f764c5ce621e356ac01df22a327ee74d1 100644
--- a/src/protocol/extensions.rs
+++ b/src/protocol/extensions.rs
@@ -18,6 +18,8 @@ pub enum Extensions {
Concatenation,
#[display(fmt = "getting")]
Getting,
+ #[display(fmt = "checksum")]
+ Checksum,
}
from_str!(Extensions, "extension");
diff --git a/src/utils/hashes.rs b/src/utils/hashes.rs
new file mode 100644
index 0000000000000000000000000000000000000000..28536ee977d49f770a58ac35dc88505537f90f00
--- /dev/null
+++ b/src/utils/hashes.rs
@@ -0,0 +1,128 @@
+use crate::{errors::RustusError, RustusResult};
+use actix_web::http::header::HeaderValue;
+use digest::Digest;
+
+/// Checks if hash-sum of a slice matches the given checksum.
+fn checksum_verify(algo: &str, bytes: &[u8], checksum: &[u8]) -> RustusResult<bool> {
+ match algo {
+ "sha1" => {
+ let sum = sha1::Sha1::digest(bytes);
+ Ok(sum.as_slice() == checksum)
+ }
+ "sha256" => {
+ let sum = sha2::Sha256::digest(bytes);
+ Ok(sum.as_slice() == checksum)
+ }
+ "sha512" => {
+ let sum = sha2::Sha512::digest(bytes);
+ Ok(sum.as_slice() == checksum)
+ }
+ "md5" => {
+ let sum = md5::Md5::digest(bytes);
+ Ok(sum.as_slice() == checksum)
+ }
+ _ => Err(RustusError::UnknownHashAlgorithm),
+ }
+}
+
+/// Verify checksum of a given chunk based on header's value.
+///
+/// This function decodes given header value.
+/// Format of the header is:
+/// <algorithm name> <base64 encoded checksum value>
+///
+/// It tries decode header value to string,
+/// splits it in two parts and after decoding base64 checksum
+/// verifies it.
+///
+/// # Errors
+///
+/// It may return error if header value can't be represented as string,
+/// if checksum can't be decoded with base64 or if unknown algorithm is used.
+pub fn verify_chunk_checksum(header: &HeaderValue, data: &[u8]) -> RustusResult<bool> {
+ if let Ok(val) = header.to_str() {
+ let mut split = val.split(' ');
+ if let Some(algo) = split.next() {
+ if let Some(checksum_base) = split.next() {
+ let checksum = base64::decode(checksum_base).map_err(|_| {
+ log::error!("Can't decode checksum value");
+ RustusError::WrongHeaderValue
+ })?;
+ return checksum_verify(algo, data, checksum.as_slice());
+ }
+ }
+ Err(RustusError::WrongHeaderValue)
+ } else {
+ log::error!("Can't decode checksum header.");
+ Err(RustusError::WrongHeaderValue)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{checksum_verify, verify_chunk_checksum};
+ use actix_web::http::header::HeaderValue;
+
+ #[test]
+ fn test_success_checksum_verify() {
+ let res = checksum_verify(
+ "sha1",
+ b"hello",
+ b"\xaa\xf4\xc6\x1d\xdc\xc5\xe8\xa2\xda\xbe\xde\x0f;H,\xd9\xae\xa9CM",
+ )
+ .unwrap();
+ assert!(res);
+ let res = checksum_verify(
+ "sha256",
+ b"hello",
+ b",\xf2M\xba_\xb0\xa3\x0e&\xe8;*\xc5\xb9\xe2\x9e\x1b\x16\x1e\\\x1f\xa7B^s\x043b\x93\x8b\x98$",
+ ).unwrap();
+ assert!(res);
+ let res = checksum_verify(
+ "sha512",
+ b"hello",
+ b"\x9bq\xd2$\xbdb\xf3x]\x96\xd4j\xd3\xea=s1\x9b\xfb\xc2\x89\x0c\xaa\xda\xe2\xdf\xf7%\x19g<\xa7##\xc3\xd9\x9b\xa5\xc1\x1d|z\xccn\x14\xb8\xc5\xda\x0cFcG\\.\\:\xde\xf4os\xbc\xde\xc0C",
+ ).unwrap();
+ assert!(res);
+ let res =
+ checksum_verify("md5", b"hello", b"]A@*\xbcK*v\xb9q\x9d\x91\x10\x17\xc5\x92").unwrap();
+ assert!(res);
+ }
+
+ #[test]
+ fn test_sum_unknown_algo_checksum_verify() {
+ let res = checksum_verify("base64", "test".as_bytes(), b"dGVzdAo=");
+ assert!(res.is_err());
+ }
+
+ #[test]
+ fn test_success_verify_chunk_checksum() {
+ let res = verify_chunk_checksum(
+ &HeaderValue::from_str("md5 XUFAKrxLKna5cZ2REBfFkg==").unwrap(),
+ b"hello",
+ )
+ .unwrap();
+ assert!(res);
+ }
+
+ #[test]
+ fn test_wrong_checksum() {
+ let res = verify_chunk_checksum(&HeaderValue::from_str("md5 memes==").unwrap(), b"hello");
+ assert!(res.is_err());
+ }
+
+ #[test]
+ fn test_bytes_header() {
+ let res = verify_chunk_checksum(
+ &HeaderValue::from_bytes(b"ewq ]A@*\xbcK*v").unwrap(),
+ b"hello",
+ );
+ assert!(res.is_err());
+ }
+
+ #[test]
+ fn test_badly_formatted_header() {
+ let res = verify_chunk_checksum(&HeaderValue::from_str("md5").unwrap(), b"hello");
+ assert!(res.is_err());
+ }
+}
diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index 0bef5e255fa47a6f53bdba0c498862fee1325665..458aa2cff946b4751d57739cd29f03b6d0d5971d 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -1,3 +1,5 @@
pub mod dir_struct;
pub mod enums;
+#[cfg(feature = "hashers")]
+pub mod hashes;
pub mod headers;