diff --git a/Cargo.lock b/Cargo.lock index 602fd875db043d77e955d7e59a82cfe2136d0c30..b7b53b6a0b57fad4f5897d2ea0673100eedcab77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2586,7 +2586,7 @@ dependencies = [ [[package]] name = "rustus" -version = "0.4.1" +version = "0.4.2" dependencies = [ "actix-files", "actix-rt", diff --git a/Cargo.toml b/Cargo.toml index a083d0debca07ad7770de7127d361a8a2172702c..c9c94887af953e0cdcdfc8b382ba88ad98c7bcbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustus" -version = "0.4.1" +version = "0.4.2" edition = "2021" description = "TUS protocol implementation written in Rust." diff --git a/README.md b/README.md index ea1bb822d14015f290251c6204c4cdf1accadfdf..db7914bf337fc9c8ce1c335fb876d34d0a922a1f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ -# Rustus - -[Tus](https://tus.io/) protocol implementation written in Rust. +<div align="center"> + <img src="./imgs/logo.svg" alt="logo" height="400"> + <div> + <p></p> + <img alt="Docker Image Size (latest by date)" src="https://img.shields.io/docker/image-size/s3rius/rustus?sort=date&style=for-the-badge"> + <img alt="Docker Image Version (latest semver)" src="https://img.shields.io/docker/v/s3rius/rustus?style=for-the-badge"> + <img alt="GitHub" src="https://img.shields.io/github/license/s3rius/rustus?style=for-the-badge"> + </div> + <p><a href="https://tus.io/">TUS</a> protocol implementation written in Rust.</p> +</div> ## Features @@ -17,11 +24,13 @@ This implementation has several features to make usage as simple as possible. You can download binaries from a [releases page](https://github.com/s3rius/rustus/releases). If you want to use docker, you can use official images from [s3rius/rustus](https://hub.docker.com/r/s3rius/rustus/): + ```bash docker run --rm -it -p 1081:1081 s3rius/rustus:latest ``` -If we don't have a binary file for your operating system you can build it with [cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html). +If we don't have a binary file for your operating system you can build it +with [cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html). ```bash git clone https://github.com/s3rius/rustus.git @@ -31,8 +40,8 @@ cargo install --path . --features=all ### Supported data storages -Right now you can only use `file-storage` to store uploads data. -The only two options you can adjust are: +Right now you can only use `file-storage` to store uploads data. The only two options you can adjust are: + * uploads directory * directory structure @@ -43,8 +52,8 @@ you can provide a `--data-dir` parameter. rustus --data-dir "./files" ``` -If you have a lot of uploads, you don't want to store all your files in -a flat structure. So you can set a directory structure for your uploads. +If you have a lot of uploads, you don't want to store all your files in a flat structure. So you can set a directory +structure for your uploads. ```bash rustus --dir-structure="{env[HOSTNAME]}/{year}/{month}/{day}" @@ -64,21 +73,18 @@ data ``` -**Important note:** if you use variable that doesn't exist or incorrect like invalid env variable, it -results in an error and the directory structure will become flat again. +**Important note:** if you use variable that doesn't exist or incorrect like invalid env variable, it results in an +error and the directory structure will become flat again. -As you can see all info files are stored in a flat structure. It cannot be changed if -you use file info storage. In order to get rid of those `.info` files use different -info storages. +As you can see all info files are stored in a flat structure. It cannot be changed if you use file info storage. In +order to get rid of those `.info` files use different info storages. ## Info storages -The info storage is a database or directory. The main goal is to keep track -of uploads. Rustus stores information about download in json format inside -database. +The info storage is a database or directory. The main goal is to keep track of uploads. Rustus stores information about +download in json format inside database. -File storage is used by default. You can customize the directory of an .info files -by providing `--info-dir` parameter. +File storage is used by default. You can customize the directory of an .info files by providing `--info-dir` parameter. ```bash rustus --info-dir "./info_dir" @@ -100,24 +106,24 @@ rustus --info-storage db-info-storage --info-db-dsn "mysql://rustus:rustus@192.1 ## Hooks Rustus supports several event hooks, such as: + * File hooks; * HTTP hooks; * AMQP hooks. -You can combine them, but you have to be careful, since -AMQP hooks won't block uploading. +You can combine them, but you have to be careful, since AMQP hooks won't block uploading. -If you want to check the "Authorization" header value or validate some information, -you have to use webhooks or File hooks. +If you want to check the "Authorization" header value or validate some information, you have to use webhooks or File +hooks. -Hooks have priorities: file hooks are the most important, then goes webhooks and AMQP hooks have the least priority. -If pre-create hook failed, the upload would not start. -Of course, since AMQP is a protocol that doesn't allow you to track responses -we can't validate anything to stop uploading. +Hooks have priorities: file hooks are the most important, then goes webhooks and AMQP hooks have the least priority. If +pre-create hook failed, the upload would not start. Of course, since AMQP is a protocol that doesn't allow you to track +responses we can't validate anything to stop uploading. Hooks can have 2 formats default: + ```json { "upload": { @@ -152,6 +158,7 @@ default: ``` tusd: + ```json { "Upload": { @@ -215,9 +222,8 @@ Rustus can work with two types of file hooks. 1. Single file hook; 2. Hooks directory. -The main difference is that hook name is passed as a command line parameter to a -single file hook, but if you use hooks directory then hook name is used to determine a -file to call. Let's take a look at the examples +The main difference is that hook name is passed as a command line parameter to a single file hook, but if you use hooks +directory then hook name is used to determine a file to call. Let's take a look at the examples Example of a single file hook: @@ -239,11 +245,13 @@ fi As you can see it uses first CLI parameter as a hook name and all hook data is received from stdin. Let's make it executable + ```bash chmod +x "hooks/unified_hook" ``` To use it you can add parameter + ```bash rustus --hooks-file "hooks/unified_hook" ``` @@ -262,29 +270,25 @@ hooks └── pre-create ``` -Every file in this directory has an executable flag. -So you can specify a parameter to use hooks directory. +Every file in this directory has an executable flag. So you can specify a parameter to use hooks directory. ```bash rustus --hooks-dir "hooks" ``` -In this case rustus will append a hook name to the directory you pointed at and call it as -an executable. +In this case rustus will append a hook name to the directory you pointed at and call it as an executable. Information about hook is passed as a first parameter, as if you call script by running: + ```bash ./hooks/pre-create '{"id": "someid", ...}' ``` ### Http Hooks -Http hooks use http protocol to notify you about an upload. -You can use HTTP hooks to verify Authorization. +Http hooks use http protocol to notify you about an upload. You can use HTTP hooks to verify Authorization. - -Let's create a FastAPI application that listens to hooks and checks the -authorization header. +Let's create a FastAPI application that listens to hooks and checks the authorization header. ```bash # Installing dependencies @@ -298,10 +302,11 @@ from typing import Optional app = FastAPI() + @app.post("/hooks") def hook( - authorization: Optional[str] = Header(None), - hook_name: Optional[str] = Header(None), + authorization: Optional[str] = Header(None), + hook_name: Optional[str] = Header(None), ): print(f"Received: {hook_name}") if authorization != "Bearer jwt": @@ -310,16 +315,17 @@ def hook( ``` Now we can start a server. + ```bash uvicorn server:app --port 8080 ``` Now you can start rustus, and it will check if Authorization header has a correct value. + ```bash rustus --hooks-http-urls "http://localhost:8000/hooks" --hooks-http-proxy-headers "Authorization" ``` - ### AMQP hooks All hooks can be sent with an AMQP protocol. @@ -335,6 +341,5 @@ This command will create an exchange called "rustus" and queues for every hook. Every hook is published with routing key "rustus.{hook_name}" like "rustus.post-create" or "rustus.pre-create" and so on. -The problem with AMQP hooks is that you can't block the upload. -To do this you have to use HTTP or File hooks. But with AMQP your -uploads become non-blocking which is definitely a good thing. \ No newline at end of file +The problem with AMQP hooks is that you can't block the upload. To do this you have to use HTTP or File hooks. But with +AMQP your uploads become non-blocking which is definitely a good thing. \ No newline at end of file diff --git a/imgs/logo.svg b/imgs/logo.svg new file mode 100644 index 0000000000000000000000000000000000000000..b1689a73c3697042c4d36053db34f87cfaa936ea --- /dev/null +++ b/imgs/logo.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 307.23 420.94"><defs><style>.cls-1{fill:#265aa6;}.cls-2{fill:url(#БезымÑнный_градиент_9);}.cls-3{fill:url(#БезымÑнный_градиент_9-2);}</style><linearGradient id="БезымÑнный_градиент_9" x1="192.38" y1="301.82" x2="192.38" y2="4.75" gradientUnits="userSpaceOnUse"><stop offset="0.25" stop-color="#46bbed"/><stop offset="0.34" stop-color="#43b2e6"/><stop offset="0.48" stop-color="#3b9ad5"/><stop offset="0.66" stop-color="#2e72b8"/><stop offset="0.75" stop-color="#265aa6"/></linearGradient><linearGradient id="БезымÑнный_градиент_9-2" x1="115.07" y1="301.82" x2="115.07" y2="4.75" xlink:href="#БезымÑнный_градиент_9"/></defs><g id="Слой_2" data-name="Слой 2"><g id="Слой_1-2" data-name="Слой 1"><path class="cls-1" d="M0,419.05v-52H14.7v6.72q6.09-7.87,20-7.66v15.54a15.47,15.47,0,0,0-7.56-1.79,11.32,11.32,0,0,0-8.71,4.1q-3.69,4.09-3.68,11.23v23.84Z"/><path class="cls-1" d="M64.16,419.68q-10.2,0-16.33-6.72t-6.14-18.9v-27h14.7v25.41q0,7,3.09,10.82a10.13,10.13,0,0,0,8.25,3.78,10.61,10.61,0,0,0,8.71-3.89q3.15-3.88,3.15-11v-25.1h14.7v52H79.8v-6.3h-.21Q73.92,419.68,64.16,419.68Z"/><path class="cls-1" d="M125,420.94A28.38,28.38,0,0,1,110.09,417a20.2,20.2,0,0,1-9-10.66l12.7-4q3,6.82,11.34,6.83a11.68,11.68,0,0,0,6.62-1.63,5.21,5.21,0,0,0,2.41-4.57,3.53,3.53,0,0,0-2.2-3.31,12.6,12.6,0,0,0-5.2-1.2q-3-.11-7.35-.69A29.41,29.41,0,0,1,112,396q-8.92-4-8.93-13.23,0-7.25,6.2-12.18a23.51,23.51,0,0,1,15.12-4.94q8.4,0,14,3.89a18.09,18.09,0,0,1,7.41,10.39l-12.5,3.47a8.65,8.65,0,0,0-3-4.89,9,9,0,0,0-5.78-1.83,9.31,9.31,0,0,0-5.72,1.68,5,5,0,0,0-2.26,4.09q0,3,3.57,4.15a50.81,50.81,0,0,0,9.66,1.68,33.21,33.21,0,0,1,9.35,1.94q9.35,3.88,9.34,13a15.23,15.23,0,0,1-6.61,12.86Q135.24,420.94,125,420.94Z"/><path class="cls-1" d="M182.49,419.26q-8.93,0-14.49-5.1t-5.56-13.49V379.88h-11V367.07h11V352h14.49v15.12h14.49v12.81H176.93v17.54q0,8.4,8.92,8.4a18.18,18.18,0,0,0,5.36-.84l1.57,12.6A34.34,34.34,0,0,1,182.49,419.26Z"/><path class="cls-1" d="M222.92,419.68q-10.18,0-16.33-6.72t-6.14-18.9v-27h14.7v25.41q0,7,3.1,10.82a10.11,10.11,0,0,0,8.24,3.78,10.6,10.6,0,0,0,8.71-3.89q3.15-3.88,3.15-11v-25.1h14.7v52H238.56v-6.3h-.21Q232.68,419.68,222.92,419.68Z"/><path class="cls-1" d="M283.71,420.94A28.32,28.32,0,0,1,268.86,417a20.18,20.18,0,0,1-9-10.66l12.7-4q3,6.82,11.34,6.83a11.68,11.68,0,0,0,6.62-1.63A5.21,5.21,0,0,0,293,403a3.54,3.54,0,0,0-2.21-3.31,12.56,12.56,0,0,0-5.2-1.2q-3-.11-7.35-.69a29.52,29.52,0,0,1-7.4-1.83q-8.93-4-8.93-13.23,0-7.25,6.2-12.18a23.51,23.51,0,0,1,15.12-4.94q8.4,0,14,3.89a18.12,18.12,0,0,1,7.4,10.39l-12.5,3.47a8.65,8.65,0,0,0-3-4.89,9,9,0,0,0-5.78-1.83,9.33,9.33,0,0,0-5.72,1.68,5,5,0,0,0-2.26,4.09q0,3,3.57,4.15a50.81,50.81,0,0,0,9.66,1.68,33.21,33.21,0,0,1,9.35,1.94q9.35,3.88,9.34,13a15.23,15.23,0,0,1-6.61,12.86Q294,420.94,283.71,420.94Z"/><path class="cls-2" d="M192.24,301.82q-41,0-63.84-23.07t-22.8-66.13V38.36c7.78-7.25,11-10.24,18.12-16.93h0c8.89-8.23,12.93-12,18.12-16.68V211.21q0,58.33,50.12,58.33,50.4,0,50.4-58.33v-92.3a18.13,18.13,0,0,1,18.12-18.13h.57a18.13,18.13,0,0,1,18.12,18.13v93.71q0,43-22.93,66.13T192.24,301.82Z"/><path class="cls-3" d="M51.21,278.75q-22.93-23.08-22.93-66.13V118.91a18.13,18.13,0,0,1,18.13-18.13H47a18.13,18.13,0,0,1,18.12,18.13v92.3q0,58.33,50.4,58.33,50.12,0,50.12-58.33V4.75c5.2,4.68,9.24,8.45,18.12,16.68h0c7.13,6.69,10.35,9.68,18.12,16.93V212.62q0,43-22.8,66.13t-63.84,23.07Q74.16,301.82,51.21,278.75Z"/><path class="cls-1" d="M88.08,96.57A17.7,17.7,0,0,1,76,65.92L141.67,4.75a17.7,17.7,0,0,1,24.12,25.9L100.14,91.82A17.62,17.62,0,0,1,88.08,96.57Z"/><path class="cls-1" d="M219.37,96.57a17.62,17.62,0,0,1-12.06-4.75L141.67,30.65a17.7,17.7,0,1,1,24.12-25.9l65.64,61.17a17.7,17.7,0,0,1-12.06,30.65Z"/></g></g></svg> \ No newline at end of file diff --git a/imgs/rustus_startup_logo.txt b/imgs/rustus_startup_logo.txt index 381d6f40ad9bfaf9d33a7f3fc9196e90fb7bfdab..098a49822a40b83ece803d669719d02e8201fd14 100644 --- a/imgs/rustus_startup_logo.txt +++ b/imgs/rustus_startup_logo.txt @@ -1,7 +1,7 @@ - ____ __ - / __ \__ _______/ /___ _______ - / /_/ / / / / ___/ __/ / / / ___/ - / _, _/ /_/ (__ ) /_/ /_/ (__ ) - /_/ |_|\__,_/____/\__/\__,_/____/ + __ + _______ _______/ /___ _______ + / ___/ / / / ___/ __/ / / / ___/ + / / / /_/ (__ ) /_/ /_/ (__ ) +/_/ \__,_/____/\__/\__,_/____/ Makes file uploads easier. diff --git a/src/info_storages/file_info_storage.rs b/src/info_storages/file_info_storage.rs index 65cd3cac2dab9699b52f982cfaad99fb33de96c4..bafa5bcfb55c4cbbf4eabbffa664ca609a192b73 100644 --- a/src/info_storages/file_info_storage.rs +++ b/src/info_storages/file_info_storage.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; use async_std::fs::{read_to_string, remove_file, DirBuilder, OpenOptions}; -use async_std::prelude::*; use async_trait::async_trait; +use futures::io::BufWriter; +use futures::AsyncWriteExt; use log::error; use crate::errors::{RustusError, RustusResult}; @@ -35,7 +36,7 @@ impl InfoStorage for FileInfoStorage { } async fn set_info(&self, file_info: &FileInfo, create: bool) -> RustusResult<()> { - let mut file = OpenOptions::new() + let file = OpenOptions::new() .write(true) .create(create) .truncate(true) @@ -45,7 +46,16 @@ impl InfoStorage for FileInfoStorage { error!("{:?}", err); RustusError::UnableToWrite(err.to_string()) })?; - file.write_all(serde_json::to_string(&file_info)?.as_bytes()) + let mut writer = BufWriter::new(file); + writer + .write_all( + serde_json::to_string(&file_info) + .map_err(|err| { + error!("{:#?}", err); + err + })? + .as_bytes(), + ) .await .map_err(|err| { error!("{:?}", err); @@ -56,7 +66,7 @@ impl InfoStorage for FileInfoStorage { .to_string(), ) })?; - file.sync_data().await?; + writer.flush().await?; Ok(()) } diff --git a/src/storages/file_storage.rs b/src/storages/file_storage.rs index ccc699c0c70ab37c155b340a412352224972f053..ba448192cf94570ed804ee0ff0d9bcd62ac4d139 100644 --- a/src/storages/file_storage.rs +++ b/src/storages/file_storage.rs @@ -3,7 +3,6 @@ use std::path::PathBuf; use actix_files::NamedFile; use async_std::fs::{remove_file, DirBuilder, File, OpenOptions}; use async_std::io::copy; -use async_std::prelude::*; use async_trait::async_trait; use log::error; @@ -12,6 +11,8 @@ use crate::info_storages::FileInfo; use crate::storages::Storage; use crate::utils::dir_struct::dir_struct; use derive_more::Display; +use futures::io::BufWriter; +use futures::AsyncWriteExt; #[derive(Display)] #[display(fmt = "file_storage")] @@ -86,7 +87,7 @@ impl Storage for FileStorage { // Opening file in w+a mode. // It means that we're going to append some // bytes to the end of a file. - let mut file = OpenOptions::new() + let file = OpenOptions::new() .write(true) .append(true) .create(false) @@ -96,12 +97,13 @@ impl Storage for FileStorage { error!("{:?}", err); RustusError::UnableToWrite(err.to_string()) })?; - file.write_all(bytes).await.map_err(|err| { + let mut writer = BufWriter::new(file); + writer.write_all(bytes).await.map_err(|err| { error!("{:?}", err); RustusError::UnableToWrite(info.path.clone().unwrap()) })?; - file.sync_data().await?; // Updating information about file. + writer.flush().await?; Ok(()) } @@ -109,9 +111,10 @@ impl Storage for FileStorage { // New path to file. let file_path = self.data_file_path(file_info.id.as_str()).await?; // Creating new file. - let mut file = OpenOptions::new() - .write(true) + OpenOptions::new() .create(true) + .write(true) + .truncate(true) .create_new(true) .open(file_path.as_path()) .await @@ -119,15 +122,6 @@ impl Storage for FileStorage { error!("{:?}", err); RustusError::FileAlreadyExists(file_info.id.clone()) })?; - - // Let's write an empty string to the beginning of the file. - // Maybe remove it later. - file.write_all(b"").await.map_err(|err| { - error!("{:?}", err); - RustusError::UnableToWrite(file_path.display().to_string()) - })?; - file.sync_all().await?; - Ok(file_path.display().to_string()) }