diff --git a/src/config.rs b/src/config.rs
index f0fddef4d7fa369338eaf24582ad7fa1fa210061..9609d3a65f2111bef026ecc9994df2edab66d8b3 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -3,11 +3,15 @@ use std::path::PathBuf;
use structopt::StructOpt;
use crate::errors::RustusError;
+use crate::info_storages::AvailableInfoStores;
use crate::storages::AvailableStores;
#[derive(StructOpt, Debug, Clone)]
pub struct StorageOptions {
/// Rustus storage type.
+ ///
+ /// Storages are used to store
+ /// uploads.
#[structopt(long, short, default_value = "file_storage", env = "RUSTUS_STORAGE")]
pub storage: AvailableStores,
@@ -21,18 +25,38 @@ pub struct StorageOptions {
required_if("storage", "file_storage"),
required_if("storage", "sqlite_file_storage")
)]
- pub data: PathBuf,
+ pub data_dir: PathBuf,
+}
- /// Path to SQLite file.
+#[derive(StructOpt, Debug, Clone)]
+pub struct InfoStoreOptions {
+ /// Type of info storage.
+ ///
+ /// Info storages are used
+ /// to store information about
+ /// uploads.
///
- /// This file is used to
- /// store information about uploaded files.
+ /// This information is used in
+ /// HEAD requests.
#[structopt(
long,
- default_value = "data/info.sqlite3",
- required_if("storage", "sqlite_file_storage")
+ short,
+ default_value = "file_info_storage",
+ env = "RUSTUS_INFO_STORAGE"
)]
- pub sqlite_dsn: PathBuf,
+ pub info_storage: AvailableInfoStores,
+
+ /// Rustus info directory
+ ///
+ /// This directory is used to store .info files
+ /// for `file_info_storage`.
+ #[structopt(
+ long,
+ default_value = "./data",
+ required_if("info_storage", "file_info_storage"),
+ env = "RUSTUS_INFO_DIR"
+ )]
+ pub info_dir: PathBuf,
}
#[derive(Debug, StructOpt, Clone)]
@@ -79,6 +103,9 @@ pub struct RustusConf {
#[structopt(flatten)]
pub storage_opts: StorageOptions,
+
+ #[structopt(flatten)]
+ pub info_storage_opts: InfoStoreOptions,
}
/// Enum of available Protocol Extensions
diff --git a/src/errors.rs b/src/errors.rs
index cc269354b338119e9cd20f02d2ff884f7ade7a74..ba591c138a4bd431fcbf8c5a2c2b9b2fa6de120f 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -1,8 +1,8 @@
use std::io::{Error, ErrorKind};
+use actix_web::{HttpResponse, ResponseError};
use actix_web::dev::HttpResponseBuilder;
use actix_web::http::StatusCode;
-use actix_web::{HttpResponse, ResponseError};
pub type RustusResult<T> = Result<T, RustusError>;
@@ -26,6 +26,8 @@ pub enum RustusError {
UnableToWrite(String),
#[error("Unable to remove file {0}")]
UnableToRemove(String),
+ #[error("Unable to prepare info storage. Reason: {0}")]
+ UnableToPrepareInfoStorage(String),
#[error("Unable to prepare storage. Reason: {0}")]
UnableToPrepareStorage(String),
#[error("Unknown extension: {0}")]
diff --git a/src/info_storages/file_info.rs b/src/info_storages/file_info.rs
new file mode 100644
index 0000000000000000000000000000000000000000..208343b70f89d73edc6d724e7c270fcc89ad97a5
--- /dev/null
+++ b/src/info_storages/file_info.rs
@@ -0,0 +1,57 @@
+use std::collections::HashMap;
+
+
+use serde::{Deserialize, Serialize};
+
+/// Information about file.
+/// It has everything about stored file.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct FileInfo {
+ pub id: String,
+ pub offset: usize,
+ pub length: usize,
+ pub path: String,
+ pub created_at: i64,
+ pub deferred_size: bool,
+ pub metadata: HashMap<String, String>,
+}
+
+impl FileInfo {
+ /// Creates new `FileInfo`.
+ ///
+ /// # Params
+ ///
+ /// File info takes
+ /// `file_id` - Unique file identifier;
+ /// `file_size` - Size of a file if it's known;
+ /// `path` - local path of a file;
+ /// `initial_metadata` - meta information, that could be omitted.
+ pub fn new(
+ file_id: &str,
+ file_size: Option<usize>,
+ path: String,
+ initial_metadata: Option<HashMap<String, String>>,
+ ) -> FileInfo {
+ let id = String::from(file_id);
+ let mut length = 0;
+ let mut deferred_size = true;
+ if let Some(size) = file_size {
+ length = size;
+ deferred_size = false;
+ }
+ let metadata = match initial_metadata {
+ Some(meta) => meta,
+ None => HashMap::new(),
+ };
+
+ FileInfo {
+ id,
+ path,
+ length,
+ metadata,
+ deferred_size,
+ offset: 0,
+ created_at: chrono::Utc::now().timestamp(),
+ }
+ }
+}
diff --git a/src/info_storages/file_info_storage.rs b/src/info_storages/file_info_storage.rs
new file mode 100644
index 0000000000000000000000000000000000000000..7341d6e473c569f8263d28adeffab461bdbb4845
--- /dev/null
+++ b/src/info_storages/file_info_storage.rs
@@ -0,0 +1,87 @@
+use std::path::PathBuf;
+
+use async_std::fs::{DirBuilder, OpenOptions, read_to_string, remove_file};
+use async_std::prelude::*;
+use async_trait::async_trait;
+use log::error;
+
+use crate::errors::{RustusError, RustusResult};
+use crate::info_storages::{FileInfo, InfoStorage};
+use crate::RustusConf;
+
+pub struct FileInfoStorage {
+ app_conf: RustusConf,
+}
+
+impl FileInfoStorage {
+ pub fn new(app_conf: RustusConf) -> Self {
+ Self { app_conf }
+ }
+
+ pub fn info_file_path(&self, file_id: &str) -> PathBuf {
+ self.app_conf
+ .info_storage_opts
+ .info_dir
+ .join(format!("{}.info", file_id))
+ }
+}
+
+#[async_trait]
+impl InfoStorage for FileInfoStorage {
+ async fn prepare(&mut self) -> RustusResult<()> {
+ if !self.app_conf.info_storage_opts.info_dir.exists() {
+ DirBuilder::new()
+ .create(self.app_conf.info_storage_opts.info_dir.as_path())
+ .await
+ .map_err(|err| RustusError::UnableToPrepareInfoStorage(err.to_string()))?;
+ }
+ Ok(())
+ }
+
+ async fn set_info(&self, file_info: &FileInfo) -> RustusResult<()> {
+ let mut file = OpenOptions::new()
+ .write(true)
+ .create(true)
+ .open(self.info_file_path(file_info.id.as_str()).as_path())
+ .await
+ .map_err(|err| {
+ error!("{:?}", err);
+ RustusError::UnableToWrite(err.to_string())
+ })?;
+ file.write_all(serde_json::to_string(&file_info)?.as_bytes())
+ .await
+ .map_err(|err| {
+ error!("{:?}", err);
+ RustusError::UnableToWrite(
+ self.info_file_path(file_info.id.as_str())
+ .as_path()
+ .display()
+ .to_string(),
+ )
+ })?;
+ Ok(())
+ }
+
+ async fn get_info(&self, file_id: &str) -> RustusResult<FileInfo> {
+ let info_path = self.info_file_path(file_id);
+ if !info_path.exists() {
+ return Err(RustusError::FileNotFound);
+ }
+ let contents = read_to_string(info_path).await.map_err(|err| {
+ error!("{:?}", err);
+ RustusError::UnableToReadInfo
+ })?;
+ serde_json::from_str::<FileInfo>(contents.as_str()).map_err(RustusError::from)
+ }
+
+ async fn remove_info(&self, file_id: &str) -> RustusResult<()> {
+ let info_path = self.info_file_path(file_id);
+ if !info_path.exists() {
+ return Err(RustusError::FileNotFound);
+ }
+ remove_file(info_path).await.map_err(|err| {
+ error!("{:?}", err);
+ RustusError::UnableToRemove(String::from(file_id))
+ })
+ }
+}
diff --git a/src/info_storages/mod.rs b/src/info_storages/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..f88bce07bffd7952994abede6de4e13094982040
--- /dev/null
+++ b/src/info_storages/mod.rs
@@ -0,0 +1,53 @@
+use std::str::FromStr;
+
+use async_trait::async_trait;
+use derive_more::{Display, From};
+
+pub use file_info::FileInfo;
+
+use crate::errors::RustusResult;
+use crate::RustusConf;
+
+mod file_info;
+
+pub mod file_info_storage;
+
+
+#[derive(PartialEq, From, Display, Clone, Debug)]
+pub enum AvailableInfoStores {
+ #[display(fmt = "FileStorage")]
+ FileInfoStorage,
+}
+
+impl FromStr for AvailableInfoStores {
+ type Err = String;
+
+ fn from_str(input: &str) -> Result<Self, Self::Err> {
+ match input {
+ "file_info_storage" => Ok(AvailableInfoStores::FileInfoStorage),
+ _ => Err(String::from("Unknown storage type")),
+ }
+ }
+}
+
+impl AvailableInfoStores {
+ /// Convert `AvailableInfoStores` to the impl `InfoStorage`.
+ ///
+ /// # Params
+ /// `config` - Rustus configuration.
+ ///
+ pub fn get(&self, config: &RustusConf) -> Box<dyn InfoStorage + Sync + Send> {
+ #[allow(clippy::single_match)]
+ match self {
+ Self::FileInfoStorage => Box::new(file_info_storage::FileInfoStorage::new(config.clone())),
+ }
+ }
+}
+
+#[async_trait]
+pub trait InfoStorage {
+ async fn prepare(&mut self) -> RustusResult<()>;
+ async fn set_info(&self, file_info: &FileInfo) -> RustusResult<()>;
+ async fn get_info(&self, file_id: &str) -> RustusResult<FileInfo>;
+ async fn remove_info(&self, file_id: &str) -> RustusResult<()>;
+}
diff --git a/src/main.rs b/src/main.rs
index 5fe173023689cfa5cac512c914b5bddebfd878e1..4a71395976e2e739c67d9036276df6fb978271c8 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,11 +1,11 @@
use std::str::FromStr;
use std::sync::Arc;
-use actix_web::http::Method;
use actix_web::{
- dev::{Server, Service},
- middleware, web, App, HttpServer,
+ App,
+ dev::{Server, Service}, HttpServer, middleware, web,
};
+use actix_web::http::Method;
use log::{error, info};
use config::RustusConf;
@@ -14,6 +14,7 @@ use crate::storages::Storage;
mod config;
mod errors;
+mod info_storages;
mod protocol;
mod routes;
mod storages;
@@ -82,7 +83,7 @@ pub fn create_server(
// It returns 404 status_code.
.default_service(web::route().to(routes::not_found))
})
- .bind((host, port))?;
+ .bind((host, port))?;
// If custom workers count variable is provided.
if let Some(workers_count) = workers {
@@ -98,7 +99,9 @@ async fn main() -> std::io::Result<()> {
let app_conf = RustusConf::from_args();
simple_logging::log_to_stderr(app_conf.log_level);
- let mut storage = app_conf.storage_opts.storage.get(&app_conf);
+ let mut info_storage = app_conf.info_storage_opts.info_storage.get(&app_conf);
+ info_storage.prepare().await?;
+ let mut storage = app_conf.storage_opts.storage.get(&app_conf, info_storage);
if let Err(err) = storage.prepare().await {
error!("{}", err);
return Err(err.into());
diff --git a/src/storages/file_storage.rs b/src/storages/file_storage.rs
index a9caf166ec5449e6e93c32b1434c58f3b706f1a9..75b2628b6488914783ad9e6d359eae094866d252 100644
--- a/src/storages/file_storage.rs
+++ b/src/storages/file_storage.rs
@@ -2,44 +2,44 @@ use std::collections::HashMap;
use std::path::PathBuf;
use actix_files::NamedFile;
-use async_std::fs::{read_to_string, remove_file, DirBuilder, OpenOptions};
+use async_std::fs::{DirBuilder, OpenOptions, remove_file};
use async_std::prelude::*;
use async_trait::async_trait;
use log::error;
use uuid::Uuid;
use crate::errors::{RustusError, RustusResult};
-use crate::storages::{FileInfo, Storage};
+use crate::info_storages::{FileInfo, InfoStorage};
use crate::RustusConf;
+use crate::storages::Storage;
-#[derive(Clone)]
pub struct FileStorage {
app_conf: RustusConf,
+ info_storage: Box<dyn InfoStorage + Send + Sync>,
}
impl FileStorage {
- pub fn new(app_conf: RustusConf) -> FileStorage {
- FileStorage { app_conf }
+ pub fn new(app_conf: RustusConf, info_storage: Box<dyn InfoStorage + Send + Sync>) -> FileStorage {
+ FileStorage {
+ app_conf,
+ info_storage,
+ }
}
- pub fn info_file_path(&self, file_id: &str) -> PathBuf {
+ pub fn data_file_path(&self, file_id: &str) -> PathBuf {
self.app_conf
.storage_opts
- .data
- .join(format!("{}.info", file_id))
- }
-
- pub fn data_file_path(&self, file_id: &str) -> PathBuf {
- self.app_conf.storage_opts.data.join(file_id.to_string())
+ .data_dir
+ .join(file_id.to_string())
}
}
#[async_trait]
impl Storage for FileStorage {
async fn prepare(&mut self) -> RustusResult<()> {
- if !self.app_conf.storage_opts.data.exists() {
+ if !self.app_conf.storage_opts.data_dir.exists() {
DirBuilder::new()
- .create(self.app_conf.storage_opts.data.as_path())
+ .create(self.app_conf.storage_opts.data_dir.as_path())
.await
.map_err(|err| RustusError::UnableToPrepareStorage(err.to_string()))?;
}
@@ -47,39 +47,7 @@ impl Storage for FileStorage {
}
async fn get_file_info(&self, file_id: &str) -> RustusResult<FileInfo> {
- let info_path = self.info_file_path(file_id);
- if !info_path.exists() {
- return Err(RustusError::FileNotFound);
- }
- let contents = read_to_string(info_path).await.map_err(|err| {
- error!("{:?}", err);
- RustusError::UnableToReadInfo
- })?;
- serde_json::from_str::<FileInfo>(contents.as_str()).map_err(RustusError::from)
- }
-
- async fn set_file_info(&self, file_info: &FileInfo) -> RustusResult<()> {
- let mut file = OpenOptions::new()
- .write(true)
- .create(true)
- .open(self.info_file_path(file_info.id.as_str()).as_path())
- .await
- .map_err(|err| {
- error!("{:?}", err);
- RustusError::UnableToWrite(err.to_string())
- })?;
- file.write_all(serde_json::to_string(&file_info)?.as_bytes())
- .await
- .map_err(|err| {
- error!("{:?}", err);
- RustusError::UnableToWrite(
- self.info_file_path(file_info.id.as_str())
- .as_path()
- .display()
- .to_string(),
- )
- })?;
- Ok(())
+ self.info_storage.get_info(file_id).await
}
async fn get_contents(&self, file_id: &str) -> RustusResult<NamedFile> {
@@ -95,7 +63,7 @@ impl Storage for FileStorage {
request_offset: usize,
bytes: &[u8],
) -> RustusResult<usize> {
- let mut info = self.get_file_info(file_id).await?;
+ let mut info = self.info_storage.get_info(file_id).await?;
if info.offset != request_offset {
return Err(RustusError::WrongOffset);
}
@@ -114,7 +82,7 @@ impl Storage for FileStorage {
RustusError::UnableToWrite(self.data_file_path(file_id).as_path().display().to_string())
})?;
info.offset += bytes.len();
- self.set_file_info(&info).await?;
+ self.info_storage.set_info(&info).await?;
Ok(info.offset)
}
@@ -157,24 +125,17 @@ impl Storage for FileStorage {
metadata,
);
- self.set_file_info(&file_info).await?;
+ self.info_storage.set_info(&file_info).await?;
Ok(file_id)
}
async fn remove_file(&self, file_id: &str) -> RustusResult<()> {
- let info_path = self.info_file_path(file_id);
- if !info_path.exists() {
- return Err(RustusError::FileNotFound);
- }
+ self.info_storage.remove_info(file_id).await?;
let data_path = self.data_file_path(file_id);
if !data_path.exists() {
return Err(RustusError::FileNotFound);
}
- remove_file(info_path).await.map_err(|err| {
- error!("{:?}", err);
- RustusError::UnableToRemove(String::from(file_id))
- })?;
remove_file(data_path).await.map_err(|err| {
error!("{:?}", err);
RustusError::UnableToRemove(String::from(file_id))
diff --git a/src/storages/mod.rs b/src/storages/mod.rs
index 9b11b3c560c5a2f94c5f2a1052fd6e3b6ba0d305..2a5cac9761557ecfb577a44a696418820f686e36 100644
--- a/src/storages/mod.rs
+++ b/src/storages/mod.rs
@@ -3,25 +3,22 @@ use std::str::FromStr;
use actix_files::NamedFile;
use async_trait::async_trait;
-use chrono::serde::ts_seconds;
-use chrono::{DateTime, Utc};
+
+
use derive_more::{Display, From};
-use serde::{Deserialize, Serialize};
-use sqlx::FromRow;
+
use crate::errors::RustusResult;
+use crate::info_storages::{FileInfo, InfoStorage};
use crate::RustusConf;
pub mod file_storage;
-pub mod sqlite_file_storage;
/// Enum of available Storage implementations.
#[derive(PartialEq, From, Display, Clone, Debug)]
pub enum AvailableStores {
#[display(fmt = "FileStorage")]
FileStorage,
- #[display(fmt = "SqliteFileStorage")]
- SqliteFileStorage,
}
impl FromStr for AvailableStores {
@@ -35,7 +32,6 @@ impl FromStr for AvailableStores {
fn from_str(input: &str) -> Result<AvailableStores, Self::Err> {
match input {
"file_storage" => Ok(AvailableStores::FileStorage),
- "sqlite_file_storage" => Ok(AvailableStores::SqliteFileStorage),
_ => Err(String::from("Unknown storage type")),
}
}
@@ -47,66 +43,14 @@ impl AvailableStores {
/// # Params
/// `config` - Rustus configuration.
///
- pub fn get(&self, config: &RustusConf) -> Box<dyn Storage + Send + Sync> {
+ pub fn get(
+ &self,
+ config: &RustusConf,
+ info_storage: Box<dyn InfoStorage + Sync + Send>,
+ ) -> Box<dyn Storage + Send + Sync> {
+ #[allow(clippy::single_match)]
match self {
- Self::FileStorage => Box::new(file_storage::FileStorage::new(config.clone())),
- Self::SqliteFileStorage => {
- Box::new(sqlite_file_storage::SQLiteFileStorage::new(config.clone()))
- }
- }
- }
-}
-
-/// Information about file.
-/// It has everything about stored file.
-#[derive(Clone, Debug, Serialize, Deserialize, FromRow)]
-pub struct FileInfo {
- pub id: String,
- pub offset: usize,
- pub length: usize,
- pub path: String,
- #[serde(with = "ts_seconds")]
- pub created_at: DateTime<Utc>,
- pub deferred_size: bool,
- pub metadata: HashMap<String, String>,
-}
-
-impl FileInfo {
- /// Creates new `FileInfo`.
- ///
- /// # Params
- ///
- /// File info takes
- /// `file_id` - Unique file identifier;
- /// `file_size` - Size of a file if it's known;
- /// `path` - local path of a file;
- /// `initial_metadata` - meta information, that could be omitted.
- pub fn new(
- file_id: &str,
- file_size: Option<usize>,
- path: String,
- initial_metadata: Option<HashMap<String, String>>,
- ) -> FileInfo {
- let id = String::from(file_id);
- let mut length = 0;
- let mut deferred_size = true;
- if let Some(size) = file_size {
- length = size;
- deferred_size = false;
- }
- let metadata = match initial_metadata {
- Some(meta) => meta,
- None => HashMap::new(),
- };
-
- FileInfo {
- id,
- path,
- length,
- metadata,
- deferred_size,
- offset: 0,
- created_at: chrono::Utc::now(),
+ Self::FileStorage => Box::new(file_storage::FileStorage::new(config.clone(), info_storage)),
}
}
}
@@ -128,14 +72,6 @@ pub trait Storage {
/// `file_id` - unique file identifier.
async fn get_file_info(&self, file_id: &str) -> RustusResult<FileInfo>;
- /// Set file info
- ///
- /// This method will save information about current upload.
- ///
- /// # Params
- /// `file_info` - information about current upload.
- async fn set_file_info(&self, file_info: &FileInfo) -> RustusResult<()>;
-
/// Get contents of a file.
///
/// This method must return NamedFile since it
diff --git a/src/storages/sqlite_file_storage.rs b/src/storages/sqlite_file_storage.rs
deleted file mode 100644
index 94ee774f8d6d3449d5fd59b1d7c050607a99ef4b..0000000000000000000000000000000000000000
--- a/src/storages/sqlite_file_storage.rs
+++ /dev/null
@@ -1,119 +0,0 @@
-use std::collections::HashMap;
-
-use actix_files::NamedFile;
-use async_std::fs::{DirBuilder, File};
-use async_trait::async_trait;
-use log::error;
-use sqlx::sqlite::SqlitePoolOptions;
-use sqlx::SqlitePool;
-use thiserror::private::PathAsDisplay;
-
-use crate::errors::{RustusError, RustusResult};
-use crate::storages::{FileInfo, Storage};
-use crate::RustusConf;
-
-#[derive(Clone)]
-pub struct SQLiteFileStorage {
- app_conf: RustusConf,
- pool: Option<SqlitePool>,
-}
-
-impl SQLiteFileStorage {
- pub fn new(app_conf: RustusConf) -> SQLiteFileStorage {
- SQLiteFileStorage {
- app_conf,
- pool: None,
- }
- }
-
- #[allow(dead_code)]
- pub fn get_pool(&self) -> RustusResult<&SqlitePool> {
- if let Some(pool) = &self.pool {
- Ok(pool)
- } else {
- error!("Pool doesn't exist.");
- Err(RustusError::Unknown)
- }
- }
-}
-
-#[async_trait]
-impl Storage for SQLiteFileStorage {
- async fn prepare(&mut self) -> RustusResult<()> {
- if !self.app_conf.storage_opts.data.exists() {
- DirBuilder::new()
- .create(self.app_conf.storage_opts.data.as_path())
- .await
- .map_err(|err| RustusError::UnableToPrepareStorage(err.to_string()))?;
- }
- if !self.app_conf.storage_opts.sqlite_dsn.exists() {
- File::create(self.app_conf.storage_opts.sqlite_dsn.clone())
- .await
- .map_err(|err| RustusError::UnableToPrepareStorage(err.to_string()))?;
- }
- let pool = SqlitePoolOptions::new()
- .max_connections(10)
- .connect(
- format!(
- "sqlite://{}",
- self.app_conf
- .storage_opts
- .sqlite_dsn
- .as_display()
- .to_string()
- )
- .as_str(),
- )
- .await
- .map_err(RustusError::from)?;
- sqlx::query(
- "CREATE TABLE IF NOT EXISTS \
- fileinfo(\
- id VARCHAR(40) PRIMARY KEY, \
- offset UNSIGNED BIG INT NOT NULL DEFAULT 0, \
- length UNSIGNED BIG INT, \
- path TEXT, \
- created_at DATETIME, \
- deferred_size BOOLEAN, \
- metadata TEXT\
- );",
- )
- .execute(&pool)
- .await?;
- self.pool = Some(pool);
- Ok(())
- }
-
- async fn get_file_info(&self, _file_id: &str) -> RustusResult<FileInfo> {
- todo!()
- }
-
- async fn set_file_info(&self, _file_info: &FileInfo) -> RustusResult<()> {
- todo!()
- }
-
- async fn get_contents(&self, _file_id: &str) -> RustusResult<NamedFile> {
- todo!()
- }
-
- async fn add_bytes(
- &self,
- _file_id: &str,
- _request_offset: usize,
- _bytes: &[u8],
- ) -> RustusResult<usize> {
- todo!()
- }
-
- async fn create_file(
- &self,
- _file_size: Option<usize>,
- _metadata: Option<HashMap<String, String>>,
- ) -> RustusResult<String> {
- todo!()
- }
-
- async fn remove_file(&self, _file_id: &str) -> RustusResult<()> {
- todo!()
- }
-}