return correct mime types, improve web security

This commit is contained in:
neri 2022-08-18 23:20:56 +02:00
parent e0b5a3fc65
commit 96eadb1723
7 changed files with 96 additions and 90 deletions

View File

@ -10,3 +10,5 @@ CREATE TABLE IF NOT EXISTS files (
ALTER TABLE files ADD COLUMN IF NOT EXISTS delete_on_download boolean; ALTER TABLE files ADD COLUMN IF NOT EXISTS delete_on_download boolean;
ALTER TABLE files ALTER COLUMN delete_on_download set not null; ALTER TABLE files ALTER COLUMN delete_on_download set not null;
ALTER TABLE files ALTER COLUMN valid_till TYPE timestamptz; ALTER TABLE files ALTER COLUMN valid_till TYPE timestamptz;
ALTER TABLE files DROP COLUMN IF EXISTS kind;
ALTER TABLE files ADD COLUMN IF NOT EXISTS content_type varchar(255);

View File

@ -1,9 +1,9 @@
use futures_util::TryStreamExt; use futures_util::TryStreamExt;
use time::OffsetDateTime;
use sqlx::{postgres::PgPool, Row}; use sqlx::{postgres::PgPool, Row};
use std::cmp::max; use std::cmp::max;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use time::ext::NumericalStdDuration; use time::ext::NumericalStdDuration;
use time::OffsetDateTime;
use tokio::fs; use tokio::fs;
use tokio::sync::mpsc::Receiver; use tokio::sync::mpsc::Receiver;
use tokio::time::timeout; use tokio::time::timeout;

View File

@ -6,19 +6,19 @@ use actix_web::{
http::header::{ http::header::{
Accept, CacheControl, CacheDirective, Charset, ContentDisposition, DispositionParam, Accept, CacheControl, CacheDirective, Charset, ContentDisposition, DispositionParam,
DispositionType, Expires, ExtendedValue, Header, HeaderValue, HttpDate, TryIntoHeaderValue, DispositionType, Expires, ExtendedValue, Header, HeaderValue, HttpDate, TryIntoHeaderValue,
ACCEPT, CACHE_CONTROL, EXPIRES, VARY, ACCEPT, CACHE_CONTROL, CONTENT_TYPE, EXPIRES, VARY, X_CONTENT_TYPE_OPTIONS,
}, },
web, Error, HttpRequest, HttpResponse, web, Error, HttpRequest, HttpResponse,
}; };
use mime::{Mime, TEXT_HTML}; use mime::{Mime, APPLICATION_OCTET_STREAM, TEXT_HTML};
use sqlx::postgres::PgPool; use sqlx::postgres::PgPool;
use std::path::Path; use std::path::Path;
use time::OffsetDateTime; use time::OffsetDateTime;
use tokio::fs; use tokio::fs;
use url::Url; use url::Url;
use crate::config::Config;
use crate::deleter; use crate::deleter;
use crate::{config::Config, file_kind::FileKind};
const TEXT_VIEW_HTML: &str = include_str!("../template/text-view.html"); const TEXT_VIEW_HTML: &str = include_str!("../template/text-view.html");
const URL_VIEW_HTML: &str = include_str!("../template/url-view.html"); const URL_VIEW_HTML: &str = include_str!("../template/url-view.html");
@ -37,14 +37,14 @@ pub async fn download(
config: web::Data<Config>, config: web::Data<Config>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
let id = req.match_info().query("id"); let id = req.match_info().query("id");
let (file_id, file_name, valid_till, file_kind, delete) = load_file_info(id, &db).await?; let (file_id, file_name, valid_till, content_type, delete) = load_file_info(id, &db).await?;
let mut path = config.files_dir.clone(); let mut path = config.files_dir.clone();
path.push(&file_id); path.push(&file_id);
let file_mime = get_content_type(&path); let mime = Mime::from_str(&content_type).unwrap_or(APPLICATION_OCTET_STREAM);
let mut response = match get_view_type(&req, &file_kind, &file_mime, &path, delete).await { let mut response = match get_view_type(&req, &mime, &path, delete).await {
ViewType::Raw => build_file_response(false, &file_name, path, file_mime, &req).await, ViewType::Raw => build_file_response(false, &file_name, path, mime, &req).await,
ViewType::Download => build_file_response(true, &file_name, path, file_mime, &req).await, ViewType::Download => build_file_response(true, &file_name, path, mime, &req).await,
ViewType::Html => build_text_response(&path).await, ViewType::Html => build_text_response(&path).await,
}?; }?;
@ -67,7 +67,7 @@ async fn load_file_info(
db: &web::Data<sqlx::Pool<sqlx::Postgres>>, db: &web::Data<sqlx::Pool<sqlx::Postgres>>,
) -> Result<(String, String, OffsetDateTime, String, bool), Error> { ) -> Result<(String, String, OffsetDateTime, String, bool), Error> {
sqlx::query_as( sqlx::query_as(
"SELECT file_id, file_name, valid_till, kind, delete_on_download from files WHERE file_id = $1", "SELECT file_id, file_name, valid_till, content_type, delete_on_download from files WHERE file_id = $1",
) )
.bind(id) .bind(id)
.fetch_optional(db.as_ref()) .fetch_optional(db.as_ref())
@ -79,18 +79,9 @@ async fn load_file_info(
.ok_or_else(|| error::ErrorNotFound("file does not exist or has expired")) .ok_or_else(|| error::ErrorNotFound("file does not exist or has expired"))
} }
fn get_content_type(path: &Path) -> Mime {
let std_path = std::path::Path::new(path.as_os_str());
tree_magic_mini::from_filepath(std_path)
.unwrap_or("application/octet-stream")
.parse::<Mime>()
.expect("tree_magic_mini should not produce invalid mime")
}
async fn get_view_type( async fn get_view_type(
req: &HttpRequest, req: &HttpRequest,
file_kind: &str, mime: &Mime,
file_mime: &Mime,
file_path: &Path, file_path: &Path,
delete_on_download: bool, delete_on_download: bool,
) -> ViewType { ) -> ViewType {
@ -100,9 +91,7 @@ async fn get_view_type(
if req.query_string().contains("raw") { if req.query_string().contains("raw") {
return ViewType::Raw; return ViewType::Raw;
} }
let is_text = if mime.type_() != mime::TEXT {
FileKind::from_str(file_kind) == Ok(FileKind::Text) || file_mime.type_() == mime::TEXT;
if !is_text {
return ViewType::Raw; return ViewType::Raw;
} }
if get_file_size(file_path).await >= TEXT_VIEW_SIZE_LIMIT { if get_file_size(file_path).await >= TEXT_VIEW_SIZE_LIMIT {
@ -113,7 +102,7 @@ async fn get_view_type(
if accept_mime == TEXT_HTML { if accept_mime == TEXT_HTML {
return ViewType::Html; return ViewType::Html;
} }
if mime_matches(&accept_mime, file_mime) { if mime_matches(&accept_mime, mime) {
break; break;
} }
} }
@ -157,7 +146,7 @@ async fn build_file_response(
download: bool, download: bool,
file_name: &str, file_name: &str,
path: PathBuf, path: PathBuf,
content_type: Mime, mime: Mime,
req: &HttpRequest, req: &HttpRequest,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
let content_disposition = ContentDisposition { let content_disposition = ContentDisposition {
@ -173,10 +162,35 @@ async fn build_file_response(
log::error!("file could not be read {:?}", file_err); log::error!("file could not be read {:?}", file_err);
error::ErrorInternalServerError("this file should be here but could not be found") error::ErrorInternalServerError("this file should be here but could not be found")
})? })?
.set_content_type(content_type) .set_content_type(mime)
.set_content_disposition(content_disposition); .set_content_disposition(content_disposition);
Ok(file.into_response(req)) let mut response = file.into_response(req);
add_headers(req, download, &mut response);
Ok(response)
}
fn add_headers(req: &HttpRequest, download: bool, response: &mut HttpResponse) {
// if the browser is trying to fetch this resource in a secure context pretend the reponse is
// just binary data so it won't be executed
let sec_fetch_mode = req
.headers()
.get("sec-fetch-mode")
.and_then(|v| v.to_str().ok());
if !download && sec_fetch_mode.is_some() && sec_fetch_mode != Some("navigate") {
response.headers_mut().insert(
CONTENT_TYPE,
HeaderValue::from_str(APPLICATION_OCTET_STREAM.as_ref())
.expect("mime type can be encoded to header value"),
);
response
.headers_mut()
.insert(X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff"));
}
// the reponse varies based on these request headers
response
.headers_mut()
.append(VARY, HeaderValue::from_static("accept, sec-fetch-mode"));
} }
fn get_disposition_params(filename: &str) -> Vec<DispositionParam> { fn get_disposition_params(filename: &str) -> Vec<DispositionParam> {

View File

@ -1,27 +0,0 @@
use std::{fmt::Display, str::FromStr};
#[derive(Debug, PartialEq)]
pub(crate) enum FileKind {
Text,
Binary,
}
impl Display for FileKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FileKind::Text => write!(f, "text"),
FileKind::Binary => write!(f, "binary"),
}
}
}
impl FromStr for FileKind {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_lowercase().as_str() {
"text" => Ok(FileKind::Text),
"binary" => Ok(FileKind::Binary),
_ => Err(format!("unknown kind {}", s)),
}
}
}

View File

@ -2,14 +2,14 @@ mod config;
mod db; mod db;
mod deleter; mod deleter;
mod download; mod download;
mod file_kind;
mod multipart; mod multipart;
mod template; mod template;
mod upload; mod upload;
use actix_files::Files; use actix_files::Files;
use actix_web::{ use actix_web::{
middleware::{self, Logger}, http::header::{HeaderName, CONTENT_SECURITY_POLICY},
middleware::{self, DefaultHeaders, Logger},
web::{self, Data}, web::{self, Data},
App, Error, HttpResponse, HttpServer, App, Error, HttpResponse, HttpServer,
}; };
@ -18,6 +18,11 @@ use sqlx::postgres::PgPool;
use std::env; use std::env;
use tokio::{sync::mpsc::channel, task}; use tokio::{sync::mpsc::channel, task};
const DEFAULT_CSP: (HeaderName, &str) = (
CONTENT_SECURITY_POLICY,
"default-src 'none'; connect-src 'self'; img-src 'self'; media-src 'self'; font-src 'self'; script-src 'self'; style-src 'self'; object-src 'none'; base-uri 'self'; frame-src 'none'; frame-ancestors 'none'; form-action 'self';"
);
async fn not_found() -> Result<HttpResponse, Error> { async fn not_found() -> Result<HttpResponse, Error> {
Ok(HttpResponse::NotFound() Ok(HttpResponse::NotFound()
.content_type("text/plain") .content_type("text/plain")
@ -51,6 +56,7 @@ async fn main() -> std::io::Result<()> {
move || { move || {
App::new() App::new()
.wrap(Logger::new(r#"%{r}a "%r" =%s %bbytes %Tsec"#)) .wrap(Logger::new(r#"%{r}a "%r" =%s %bbytes %Tsec"#))
.wrap(DefaultHeaders::new().add(DEFAULT_CSP))
.wrap(middleware::Compress::default()) .wrap(middleware::Compress::default())
.app_data(db.clone()) .app_data(db.clone())
.app_data(expiry_watch_sender.clone()) .app_data(expiry_watch_sender.clone())

View File

@ -1,7 +1,8 @@
use crate::{config, file_kind::FileKind}; use crate::config;
use actix_multipart::{Field, Multipart}; use actix_multipart::{Field, Multipart};
use actix_web::{error, http::header::DispositionParam, Error}; use actix_web::{error, http::header::DispositionParam, Error};
use futures_util::{StreamExt, TryStreamExt}; use futures_util::{StreamExt, TryStreamExt};
use mime::{Mime, TEXT_PLAIN};
use std::path::Path; use std::path::Path;
use time::{Duration, OffsetDateTime}; use time::{Duration, OffsetDateTime};
use tokio::{fs::File, io::AsyncWriteExt}; use tokio::{fs::File, io::AsyncWriteExt};
@ -10,21 +11,20 @@ const MAX_UPLOAD_SECONDS: i64 = 31 * 24 * 60 * 60;
const DEFAULT_UPLOAD_SECONDS: u32 = 30 * 60; const DEFAULT_UPLOAD_SECONDS: u32 = 30 * 60;
pub(crate) struct UploadConfig { pub(crate) struct UploadConfig {
pub original_name: String, pub original_name: Option<String>,
pub content_type: Mime,
pub valid_till: OffsetDateTime, pub valid_till: OffsetDateTime,
pub kind: FileKind,
pub delete_on_download: bool, pub delete_on_download: bool,
} }
pub(crate) async fn parse_multipart( pub(crate) async fn parse_multipart(
mut payload: Multipart, mut payload: Multipart,
file_id: &str, file_path: &Path,
file_name: &Path,
config: &config::Config, config: &config::Config,
) -> Result<UploadConfig, error::Error> { ) -> Result<UploadConfig, error::Error> {
let mut original_name: Option<String> = None; let mut original_name: Option<String> = None;
let mut content_type: Option<Mime> = None;
let mut keep_for: Option<String> = None; let mut keep_for: Option<String> = None;
let mut kind: Option<FileKind> = None;
let mut delete_on_download = false; let mut delete_on_download = false;
let mut password = None; let mut password = None;
let mut size = 0; let mut size = 0;
@ -37,22 +37,20 @@ pub(crate) async fn parse_multipart(
keep_for = Some(parse_string(name, field).await?); keep_for = Some(parse_string(name, field).await?);
} }
"file" => { "file" => {
let file_original_name = get_original_filename(&field); let (mime, uploaded_name) = get_file_metadata(&field);
if file_original_name == None || file_original_name.map(|f| f.as_str()) == Some("") if uploaded_name == None || uploaded_name.map(|f| f.as_str()) == Some("") {
{
continue; continue;
} }
original_name = file_original_name.map(|f| f.to_string()); original_name = uploaded_name.map(|f| f.to_string());
kind = Some(FileKind::Binary); content_type = Some(mime.clone());
size = create_file(file_name, field, config.max_file_size).await?; size = create_file(file_path, field, config.max_file_size).await?;
} }
"text" => { "text" => {
if original_name.is_some() { if original_name.is_some() {
continue; continue;
} }
original_name = Some(format!("{}.txt", file_id)); size = create_file(file_path, field, config.max_file_size).await?;
kind = Some(FileKind::Text); content_type = Some(get_content_type(file_path));
size = create_file(file_name, field, config.max_file_size).await?;
} }
"delete_on_download" => { "delete_on_download" => {
delete_on_download = parse_string(name, field).await? != "false"; delete_on_download = parse_string(name, field).await? != "false";
@ -64,8 +62,8 @@ pub(crate) async fn parse_multipart(
}; };
} }
let original_name = original_name.ok_or_else(|| error::ErrorBadRequest("no content found"))?; let content_type =
let kind = kind.ok_or_else(|| error::ErrorBadRequest("no content found"))?; content_type.ok_or_else(|| error::ErrorBadRequest("no content type found"))?;
let keep_for: u32 = keep_for let keep_for: u32 = keep_for
.map(|k| k.parse()) .map(|k| k.parse())
.transpose() .transpose()
@ -76,8 +74,8 @@ pub(crate) async fn parse_multipart(
let upload_config = UploadConfig { let upload_config = UploadConfig {
original_name, original_name,
content_type,
valid_till, valid_till,
kind,
delete_on_download, delete_on_download,
}; };
@ -93,9 +91,11 @@ fn check_requirements(
valid_duration: &Duration, valid_duration: &Duration,
config: &config::Config, config: &config::Config,
) -> Result<(), error::Error> { ) -> Result<(), error::Error> {
if upload_config.original_name.len() > 255 { if let Some(original_name) = upload_config.original_name.as_ref() {
if original_name.len() > 255 {
return Err(error::ErrorBadRequest("filename is too long")); return Err(error::ErrorBadRequest("filename is too long"));
} }
}
let valid_seconds = valid_duration.whole_seconds(); let valid_seconds = valid_duration.whole_seconds();
if valid_seconds > MAX_UPLOAD_SECONDS { if valid_seconds > MAX_UPLOAD_SECONDS {
@ -180,13 +180,22 @@ async fn write_to_file(
Ok(written_bytes) Ok(written_bytes)
} }
fn get_original_filename(field: &actix_multipart::Field) -> Option<&String> { fn get_file_metadata(field: &actix_multipart::Field) -> (&Mime, Option<&String>) {
field let mime = field.content_type();
let filename = field
.content_disposition() .content_disposition()
.parameters .parameters
.iter() .iter()
.find_map(|param| match param { .find_map(|param| match param {
DispositionParam::Filename(filename) => Some(filename), DispositionParam::Filename(filename) => Some(filename),
_ => None, _ => None,
}) });
(mime, filename)
}
fn get_content_type(path: &Path) -> Mime {
let std_path = std::path::Path::new(path.as_os_str());
tree_magic_mini::from_filepath(std_path)
.and_then(|mime| mime.parse().ok())
.unwrap_or(TEXT_PLAIN)
} }

View File

@ -1,7 +1,6 @@
use std::io::ErrorKind; use std::io::ErrorKind;
use crate::config::Config; use crate::config::Config;
use crate::file_kind::FileKind;
use crate::multipart::UploadConfig; use crate::multipart::UploadConfig;
use crate::{multipart, template}; use crate::{multipart, template};
use actix_files::NamedFile; use actix_files::NamedFile;
@ -42,11 +41,11 @@ pub async fn upload(
error::ErrorInternalServerError("could not create file") error::ErrorInternalServerError("could not create file")
})?; })?;
let parsed_multipart = multipart::parse_multipart(payload, &file_id, &file_name, &config).await; let parsed_multipart = multipart::parse_multipart(payload, &file_name, &config).await;
let UploadConfig { let UploadConfig {
original_name, original_name,
content_type,
valid_till, valid_till,
kind,
delete_on_download, delete_on_download,
} = match parsed_multipart { } = match parsed_multipart {
Ok(data) => data, Ok(data) => data,
@ -65,14 +64,17 @@ pub async fn upload(
} }
}; };
let file_name = original_name
.clone()
.unwrap_or_else(|| format!("{}.txt", file_id));
let db_insert = sqlx::query( let db_insert = sqlx::query(
"INSERT INTO Files (file_id, file_name, valid_till, kind, delete_on_download) \ "INSERT INTO Files (file_id, file_name, content_type, valid_till, delete_on_download) \
VALUES ($1, $2, $3, $4, $5)", VALUES ($1, $2, $3, $4, $5)",
) )
.bind(&file_id) .bind(&file_id)
.bind(&original_name) .bind(&file_name)
.bind(&content_type.to_string())
.bind(valid_till) .bind(valid_till)
.bind(kind.to_string())
.bind(delete_on_download) .bind(delete_on_download)
.execute(db.as_ref()) .execute(db.as_ref())
.await; .await;
@ -90,24 +92,24 @@ pub async fn upload(
} }
log::info!( log::info!(
"{} create new file {} (valid_till: {}, kind: {}, delete_on_download: {})", "{} create new file {} (valid_till: {}, content_type: {}, delete_on_download: {})",
req.connection_info().realip_remote_addr().unwrap_or("-"), req.connection_info().realip_remote_addr().unwrap_or("-"),
file_id, file_id,
valid_till, valid_till,
kind, content_type,
delete_on_download delete_on_download
); );
expiry_watch_sender.send(()).await.unwrap(); expiry_watch_sender.send(()).await.unwrap();
let redirect = if kind == FileKind::Binary { let redirect = if let Some(original_name) = original_name.as_ref() {
let encoded_name = urlencoding::encode(&original_name); let encoded_name = urlencoding::encode(original_name);
format!("/upload/{}/{}", file_id, encoded_name) format!("/upload/{}/{}", file_id, encoded_name)
} else { } else {
format!("/upload/{}", file_id) format!("/upload/{}", file_id)
}; };
let url = get_file_url(&req, &file_id, Some(&original_name)); let url = get_file_url(&req, &file_id, original_name.as_deref());
Ok(HttpResponse::SeeOther() Ok(HttpResponse::SeeOther()
.insert_header((LOCATION, redirect)) .insert_header((LOCATION, redirect))
.body(format!("{}\n", url))) .body(format!("{}\n", url)))