diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a87a93da251c648ab0f2374e304067d56c95664..14f6fe7f711f8c720f58bc116270836810134763 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,8 +38,12 @@ repos: - clippy - --features=all - --all + - -p + - rustus - -- - -W - clippy::all - -W - - clippy::pedantic \ No newline at end of file + - clippy::pedantic + - -D + - warnings \ No newline at end of file diff --git a/src/errors.rs b/src/errors.rs index 90563739a1f4708605b641ed3d2d1c2788ae4129..7dd6475c8eb4638953ec11d49c0e88500c04431d 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -17,6 +17,8 @@ pub enum RustusError { Unknown, #[error("File is frozen")] FrozenFile, + #[error("Size already known")] + SizeAlreadyKnown, #[error("Unable to serialize object")] UnableToSerialize(#[from] serde_json::Error), #[cfg(feature = "db_info_storage")] @@ -68,7 +70,9 @@ impl ResponseError for RustusError { match self { RustusError::FileNotFound => StatusCode::NOT_FOUND, RustusError::WrongOffset => StatusCode::CONFLICT, - RustusError::FrozenFile | RustusError::HookError(_) => StatusCode::BAD_REQUEST, + RustusError::FrozenFile | RustusError::SizeAlreadyKnown | RustusError::HookError(_) => { + StatusCode::BAD_REQUEST + } _ => StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/src/info_storages/models/file_info.rs b/src/info_storages/models/file_info.rs index 154984eade7f0605f4599aa5feea972074738938..bcab553c9a387a80313d6a9b07949587848a1f99 100644 --- a/src/info_storages/models/file_info.rs +++ b/src/info_storages/models/file_info.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; pub struct FileInfo { pub id: String, pub offset: usize, - pub length: usize, + pub length: Option<usize>, pub path: Option<String>, #[serde(with = "ts_seconds")] pub created_at: DateTime<Utc>, @@ -30,15 +30,14 @@ impl FileInfo { /// `initial_metadata` - meta information, that could be omitted. pub fn new( file_id: &str, - file_size: Option<usize>, + length: Option<usize>, path: Option<String>, initial_metadata: Option<HashMap<String, String>>, ) -> FileInfo { let id = String::from(file_id); - let mut length = 0; + let mut deferred_size = true; - if let Some(size) = file_size { - length = size; + if length.is_some() { deferred_size = false; } let metadata = match initial_metadata { @@ -56,4 +55,27 @@ impl FileInfo { created_at: chrono::Utc::now(), } } + + /// Function to construct `String` value + /// from file metadata `HashMap`. + /// + /// This algorithm can be found at + /// [protocol page](https://tus.io/protocols/resumable-upload.html#upload-metadata). + pub fn get_metadata_string(&self) -> Option<String> { + let mut result = Vec::new(); + + // Getting all metadata keys. + for (key, val) in &self.metadata { + let encoded_value = base64::encode(val); + // Adding metadata entry to the list. + result.push(format!("{} {}", key, encoded_value)); + } + + if result.is_empty() { + None + } else { + // Merging the metadata. + Some(result.join(",")) + } + } } diff --git a/src/notifiers/models/message_format.rs b/src/notifiers/models/message_format.rs index d64019240df64b96401df6e11eb09adde2b18a35..578c74cd5fec5194f28fbef2b2790f1c785eab1b 100644 --- a/src/notifiers/models/message_format.rs +++ b/src/notifiers/models/message_format.rs @@ -46,20 +46,23 @@ struct TusdFileInfo { #[serde(rename = "ID")] id: String, offset: usize, - size: usize, + size: Option<usize>, is_final: bool, is_partial: bool, partial_uploads: Option<Vec<String>>, + size_is_deferred: bool, metadata: HashMap<String, String>, storage: TusdStorageInfo, } impl From<FileInfo> for TusdFileInfo { fn from(file_info: FileInfo) -> Self { + let deferred_size = file_info.length.is_none(); Self { id: file_info.id, offset: file_info.offset, size: file_info.length, + size_is_deferred: deferred_size, is_final: true, is_partial: false, partial_uploads: None, diff --git a/src/protocol/core/routes.rs b/src/protocol/core/routes.rs index 8fe8366bf1cd17a76fae50070cf9dd1e689310ff..cdb0ecbbde02fc7768a44ead7fd5d39d5d8f8e6a 100644 --- a/src/protocol/core/routes.rs +++ b/src/protocol/core/routes.rs @@ -1,6 +1,7 @@ use actix_web::{http::StatusCode, web, web::Bytes, HttpRequest, HttpResponse}; use crate::notifiers::Hook; +use crate::protocol::extensions::Extensions; use crate::utils::headers::{check_header, parse_header}; use crate::{NotificationManager, RustusConf, Storage}; @@ -26,11 +27,16 @@ pub async fn get_file_info( let file_info = storage.get_file_info(file_id).await?; builder .insert_header(("Upload-Offset", file_info.offset.to_string())) - .insert_header(("Upload-Length", file_info.length.to_string())) .insert_header(("Content-Length", file_info.offset.to_string())); - if file_info.deferred_size { + // Upload length is known. + if let Some(upload_len) = file_info.length { + builder.insert_header(("Upload-Length", upload_len.to_string())); + } else { builder.insert_header(("Upload-Defer-Length", "1")); } + if let Some(meta) = file_info.get_metadata_string() { + builder.insert_header(("Upload-Metadata", meta)); + } } else { builder.status(StatusCode::NOT_FOUND); }; @@ -53,12 +59,23 @@ pub async fn write_bytes( return Ok(HttpResponse::UnsupportedMediaType().body("")); } + // New upload length. + // Parses header `Upload-Length` only if the creation-defer-length extension is enabled. + let updated_len = if app_conf + .extensions_vec() + .contains(&Extensions::CreationDeferLength) + { + parse_header(&request, "Upload-Length") + } else { + None + }; + if let Some(file_id) = request.match_info().get("file_id") { let file_info = storage - .add_bytes(file_id, offset.unwrap(), bytes.as_ref()) + .add_bytes(file_id, offset.unwrap(), updated_len, bytes.as_ref()) .await?; let mut hook = Hook::PostReceive; - if file_info.length == file_info.offset { + if file_info.length == Some(file_info.offset) { hook = Hook::PostFinish; } if app_conf.hook_is_active(hook) { diff --git a/src/protocol/creation/routes.rs b/src/protocol/creation/routes.rs index 79079dd08b6bea17083e23936c91423d18ed897f..59375a08e911ef8ca7a0981ed14bc6b608e8c78c 100644 --- a/src/protocol/creation/routes.rs +++ b/src/protocol/creation/routes.rs @@ -102,7 +102,7 @@ pub async fn create_file( } // Writing first bytes. file_info = storage - .add_bytes(file_info.id.as_str(), 0, bytes.as_ref()) + .add_bytes(file_info.id.as_str(), 0, None, bytes.as_ref()) .await?; } @@ -111,6 +111,8 @@ pub async fn create_file( .notification_opts .notification_format .format(&request, &file_info)?; + // Adding send_message task to tokio reactor. + // Thin function would be executed in background. tokio::spawn(async move { notification_manager .send_message(message, Hook::PostCreate) diff --git a/src/storages/file_storage.rs b/src/storages/file_storage.rs index bad2c39cd648e0327713e31249d0d9ce014a6c0a..a8dbd6e37c379591c5c51033c0075f154517ee66 100644 --- a/src/storages/file_storage.rs +++ b/src/storages/file_storage.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::path::PathBuf; use actix_files::NamedFile; -use async_std::fs::create_dir_all; use async_std::fs::{remove_file, DirBuilder, OpenOptions}; use async_std::prelude::*; use async_trait::async_trait; @@ -35,16 +34,21 @@ impl FileStorage { .app_conf .storage_opts .data_dir + // We're working wit absolute paths, because tus.io says so. .canonicalize() .map_err(|err| { error!("{}", err); RustusError::UnableToWrite(err.to_string()) })? .join(self.app_conf.dir_struct().as_str()); - create_dir_all(dir.as_path()).await.map_err(|err| { - error!("{}", err); - RustusError::UnableToWrite(err.to_string()) - })?; + DirBuilder::new() + .recursive(true) + .create(dir.as_path()) + .await + .map_err(|err| { + error!("{}", err); + RustusError::UnableToWrite(err.to_string()) + })?; Ok(dir.join(file_id.to_string())) } } @@ -52,8 +56,11 @@ impl FileStorage { #[async_trait] impl Storage for FileStorage { async fn prepare(&mut self) -> RustusResult<()> { + // We're creating directory for new files + // if it doesn't already exist. if !self.app_conf.storage_opts.data_dir.exists() { DirBuilder::new() + .recursive(true) .create(self.app_conf.storage_opts.data_dir.as_path()) .await .map_err(|err| RustusError::UnableToPrepareStorage(err.to_string()))?; @@ -62,6 +69,7 @@ impl Storage for FileStorage { } async fn get_file_info(&self, file_id: &str) -> RustusResult<FileInfo> { + // I don't think comments are convenient here. self.info_storage.get_info(file_id).await } @@ -82,18 +90,46 @@ impl Storage for FileStorage { &self, file_id: &str, request_offset: usize, + updated_length: Option<usize>, bytes: &[u8], ) -> RustusResult<FileInfo> { let mut info = self.info_storage.get_info(file_id).await?; + // Checking that provided offset is equal to offset provided by request. if info.offset != request_offset { return Err(RustusError::WrongOffset); } + // In normal situation this `if` statement is not + // gonna be called, but what if it is ... if info.path.is_none() { return Err(RustusError::FileNotFound); } - if info.offset == info.length { + // This thing is only applicable in case + // if tus-extension `creation-defer-length` is enabled. + if let Some(new_len) = updated_length { + // Whoop, someone gave us total file length + // less that he had already uploaded. + if new_len < info.offset { + return Err(RustusError::WrongOffset); + } + // We already know the exact size of a file. + // Someone want to update it. + // Anyway, it's not allowed, heh. + if info.length.is_some() { + return Err(RustusError::SizeAlreadyKnown); + } + + // All checks are ok. Now our file will have exact size. + info.deferred_size = false; + info.length = Some(new_len); + } + // Checking if the size of the upload is already equals + // to calculated offset. It means that all bytes were already written. + if Some(info.offset) == info.length { return Err(RustusError::FrozenFile); } + // Opening file in w+a mode. + // It means that we're going to append some + // bytes to the end of a file. let mut file = OpenOptions::new() .write(true) .append(true) @@ -108,6 +144,7 @@ impl Storage for FileStorage { error!("{:?}", err); RustusError::UnableToWrite(info.path.clone().unwrap()) })?; + // Updating information about file. info.offset += bytes.len(); self.info_storage.set_info(&info, false).await?; Ok(info) @@ -118,8 +155,14 @@ impl Storage for FileStorage { file_size: Option<usize>, metadata: Option<HashMap<String, String>>, ) -> RustusResult<FileInfo> { + // Let's create a new file ID. + // I guess the algo for generating new upload-id's can be + // configurable. But for now I don't really care, since UUIv4 works fine. + // Maybe update it later. let file_id = Uuid::new_v4().simple().to_string(); + // New path to file. let file_path = self.data_file_path(file_id.as_str()).await?; + // Creating new file. let mut file = OpenOptions::new() .write(true) .create(true) @@ -131,12 +174,14 @@ impl Storage for FileStorage { RustusError::FileAlreadyExists(file_id.clone()) })?; - // We write empty file here. + // Let's write an empty string to the beginning of the file. + // Maybe remove it later. file.write_all(b"").await.map_err(|err| { error!("{:?}", err); RustusError::UnableToWrite(file_path.display().to_string()) })?; + // Creating new FileInfo object and saving it. let file_info = FileInfo::new( file_id.as_str(), file_size, @@ -151,13 +196,20 @@ impl Storage for FileStorage { async fn remove_file(&self, file_id: &str) -> RustusResult<FileInfo> { let info = self.info_storage.get_info(file_id).await?; + // Whoops, someone forgot to update the path field. if info.path.is_none() { return Err(RustusError::FileNotFound); } + // Let's remove info first, so file won't show up + // In get_contents function. self.info_storage.remove_info(file_id).await?; + // Let's remove the file itself. let data_path = PathBuf::from(info.path.as_ref().unwrap().clone()); if !data_path.exists() { + // Maybe we don't need error here, + // since if file doesn't exist, we're done. + // FIXME: Find it out. return Err(RustusError::FileNotFound); } remove_file(data_path).await.map_err(|err| { diff --git a/src/storages/models/storage.rs b/src/storages/models/storage.rs index 955ad34dafa949b5bcdc9c92f7d17db077292164..31df02e7e22c7765d09baba5c07f93d6c9508495 100644 --- a/src/storages/models/storage.rs +++ b/src/storages/models/storage.rs @@ -11,6 +11,10 @@ pub trait Storage { /// Function to check if configuration is correct /// and prepare storage E.G. create connection pool, /// or directory for files. + /// + /// It MUST throw errors if connection can't + /// be established or in any other case that might + /// be a problem later on. async fn prepare(&mut self) -> RustusResult<()>; /// Get file information. @@ -26,6 +30,8 @@ pub trait Storage { /// This method must return NamedFile since it /// is compatible with ActixWeb files interface. /// + /// This method basically must call info storage method. + /// /// # Params /// `file_id` - unique file identifier. async fn get_contents(&self, file_id: &str) -> RustusResult<NamedFile>; @@ -35,14 +41,25 @@ pub trait Storage { /// This method is used to append bytes to some file. /// It returns new offset. /// + /// # Errors + /// + /// Implementations MUST throw errors at following cases: + /// * Offset for request doesn't match offset we have in info storage. + /// * Updated length is provided, but we already know total size in bytes. + /// * If the info about the file can't be found. + /// * If the storage is offline. + /// /// # Params /// `file_id` - unique file identifier; /// `request_offset` - offset from the client. + /// `updated_length` - total file size in bytes. + /// This value is used by creation-defer-length extension. /// `bytes` - bytes to append to the file. async fn add_bytes( &self, file_id: &str, request_offset: usize, + updated_length: Option<usize>, bytes: &[u8], ) -> RustusResult<FileInfo>; @@ -50,6 +67,8 @@ pub trait Storage { /// /// This method is used to generate unique file id, create file and store information about it. /// + /// This function must use info storage to store information about the upload. + /// /// # Params /// `file_size` - Size of a file. It may be None if size is deferred; /// `metadata` - Optional file meta-information;