X-Git-Url: http://these/git/?a=blobdiff_plain;f=crates%2Fdb_schema%2Fsrc%2Futils.rs;h=94c867d6b897d8f6d0c920a197c803985391d36f;hb=92568956353f21649ed9aff68b42699c9d036f30;hp=c9ccc170089c3fcde6a949a908e76937dd1279b5;hpb=a2a594b7635db2241602be56250f7d9bf992f7b9;p=lemmy.git diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index c9ccc170..94c867d6 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -1,21 +1,151 @@ -use crate::newtypes::DbUrl; -use activitypub_federation::{core::object_id::ObjectId, traits::ApubObject}; +use crate::{ + diesel::Connection, + diesel_migrations::MigrationHarness, + newtypes::DbUrl, + CommentSortType, + SortType, +}; +use activitypub_federation::{fetch::object_id::ObjectId, traits::Object}; use chrono::NaiveDateTime; +use deadpool::Runtime; use diesel::{ backend::Backend, deserialize::FromSql, + pg::Pg, + result::{ConnectionError, ConnectionResult, Error as DieselError, Error::QueryBuilderError}, serialize::{Output, ToSql}, sql_types::Text, - Connection, PgConnection, }; -use lemmy_utils::error::LemmyError; +use diesel_async::{ + pg::AsyncPgConnection, + pooled_connection::{ + deadpool::{Object as PooledConnection, Pool}, + AsyncDieselConnectionManager, + }, +}; +use diesel_migrations::EmbeddedMigrations; +use futures_util::{future::BoxFuture, FutureExt}; +use lemmy_utils::{ + error::{LemmyError, LemmyErrorExt, LemmyErrorType}, + settings::structs::Settings, +}; use once_cell::sync::Lazy; use regex::Regex; -use std::{env, env::VarError, io::Write}; +use rustls::{ + client::{ServerCertVerified, ServerCertVerifier}, + ServerName, +}; +use std::{ + env, + env::VarError, + ops::{Deref, DerefMut}, + sync::Arc, + time::{Duration, SystemTime}, +}; +use tracing::{error, info}; use url::Url; -pub type DbPool = diesel::r2d2::Pool>; +const FETCH_LIMIT_DEFAULT: i64 = 10; +pub const FETCH_LIMIT_MAX: i64 = 50; +const POOL_TIMEOUT: Option = Some(Duration::from_secs(5)); + +pub type ActualDbPool = Pool; + +/// References a pool or connection. Functions must take `&mut DbPool<'_>` to allow implicit reborrowing. +/// +/// https://github.com/rust-lang/rfcs/issues/1403 +pub enum DbPool<'a> { + Pool(&'a ActualDbPool), + Conn(&'a mut AsyncPgConnection), +} + +pub enum DbConn<'a> { + Pool(PooledConnection), + Conn(&'a mut AsyncPgConnection), +} + +pub async fn get_conn<'a, 'b: 'a>(pool: &'a mut DbPool<'b>) -> Result, DieselError> { + Ok(match pool { + DbPool::Pool(pool) => DbConn::Pool(pool.get().await.map_err(|e| QueryBuilderError(e.into()))?), + DbPool::Conn(conn) => DbConn::Conn(conn), + }) +} + +impl<'a> Deref for DbConn<'a> { + type Target = AsyncPgConnection; + + fn deref(&self) -> &Self::Target { + match self { + DbConn::Pool(conn) => conn.deref(), + DbConn::Conn(conn) => conn.deref(), + } + } +} + +impl<'a> DerefMut for DbConn<'a> { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + DbConn::Pool(conn) => conn.deref_mut(), + DbConn::Conn(conn) => conn.deref_mut(), + } + } +} + +// Allows functions that take `DbPool<'_>` to be called in a transaction by passing `&mut conn.into()` +impl<'a> From<&'a mut AsyncPgConnection> for DbPool<'a> { + fn from(value: &'a mut AsyncPgConnection) -> Self { + DbPool::Conn(value) + } +} + +impl<'a, 'b: 'a> From<&'a mut DbConn<'b>> for DbPool<'a> { + fn from(value: &'a mut DbConn<'b>) -> Self { + DbPool::Conn(value.deref_mut()) + } +} + +impl<'a> From<&'a ActualDbPool> for DbPool<'a> { + fn from(value: &'a ActualDbPool) -> Self { + DbPool::Pool(value) + } +} + +/// Runs multiple async functions that take `&mut DbPool<'_>` as input and return `Result`. Only works when the `futures` crate is listed in `Cargo.toml`. +/// +/// `$pool` is the value given to each function. +/// +/// A `Result` is returned (not in a `Future`, so don't use `.await`). The `Ok` variant contains a tuple with the values returned by the given functions. +/// +/// The functions run concurrently if `$pool` has the `DbPool::Pool` variant. +#[macro_export] +macro_rules! try_join_with_pool { + ($pool:ident => ($($func:expr),+)) => {{ + // Check type + let _: &mut $crate::utils::DbPool<'_> = $pool; + + match $pool { + // Run concurrently with `try_join` + $crate::utils::DbPool::Pool(__pool) => ::futures::try_join!( + $(async { + let mut __dbpool = $crate::utils::DbPool::Pool(__pool); + ($func)(&mut __dbpool).await + }),+ + ), + // Run sequentially + $crate::utils::DbPool::Conn(__conn) => async { + Ok(($({ + let mut __dbpool = $crate::utils::DbPool::Conn(__conn); + // `?` prevents the error type from being inferred in an `async` block, so `match` is used instead + match ($func)(&mut __dbpool).await { + ::core::result::Result::Ok(__v) => __v, + ::core::result::Result::Err(__v) => return ::core::result::Result::Err(__v), + } + }),+)) + }.await, + } + }}; +} pub fn get_database_url_from_env() -> Result { env::var("LEMMY_DATABASE_URL") @@ -23,13 +153,42 @@ pub fn get_database_url_from_env() -> Result { pub fn fuzzy_search(q: &str) -> String { let replaced = q.replace('%', "\\%").replace('_', "\\_").replace(' ', "%"); - format!("%{}%", replaced) + format!("%{replaced}%") } -pub fn limit_and_offset(page: Option, limit: Option) -> (i64, i64) { - let page = page.unwrap_or(1); - let limit = limit.unwrap_or(10); +pub fn limit_and_offset( + page: Option, + limit: Option, +) -> Result<(i64, i64), diesel::result::Error> { + let page = match page { + Some(page) => { + if page < 1 { + return Err(QueryBuilderError("Page is < 1".into())); + } else { + page + } + } + None => 1, + }; + let limit = match limit { + Some(limit) => { + if !(1..=FETCH_LIMIT_MAX).contains(&limit) { + return Err(QueryBuilderError( + format!("Fetch limit is > {FETCH_LIMIT_MAX}").into(), + )); + } else { + limit + } + } + None => FETCH_LIMIT_DEFAULT, + }; let offset = limit * (page - 1); + Ok((limit, offset)) +} + +pub fn limit_and_offset_unlimited(page: Option, limit: Option) -> (i64, i64) { + let limit = limit.unwrap_or(FETCH_LIMIT_DEFAULT); + let offset = limit * (page.unwrap_or(1) - 1); (limit, offset) } @@ -42,7 +201,7 @@ pub fn diesel_option_overwrite(opt: &Option) -> Option> { // An empty string is an erase Some(unwrapped) => { if !unwrapped.eq("") { - Some(Some(unwrapped.to_owned())) + Some(Some(unwrapped.clone())) } else { Some(None) } @@ -54,44 +213,161 @@ pub fn diesel_option_overwrite(opt: &Option) -> Option> { pub fn diesel_option_overwrite_to_url( opt: &Option, ) -> Result>, LemmyError> { - match opt.as_ref().map(|s| s.as_str()) { + match opt.as_ref().map(String::as_str) { // An empty string is an erase Some("") => Ok(Some(None)), - Some(str_url) => match Url::parse(str_url) { - Ok(url) => Ok(Some(Some(url.into()))), - Err(e) => Err(LemmyError::from_error_message(e, "invalid_url")), - }, + Some(str_url) => Url::parse(str_url) + .map(|u| Some(Some(u.into()))) + .with_lemmy_type(LemmyErrorType::InvalidUrl), None => Ok(None), } } -embed_migrations!(); +pub fn diesel_option_overwrite_to_url_create( + opt: &Option, +) -> Result, LemmyError> { + match opt.as_ref().map(String::as_str) { + // An empty string is nothing + Some("") => Ok(None), + Some(str_url) => Url::parse(str_url) + .map(|u| Some(u.into())) + .with_lemmy_type(LemmyErrorType::InvalidUrl), + None => Ok(None), + } +} -pub fn establish_unpooled_connection() -> PgConnection { - let db_url = match get_database_url_from_env() { - Ok(url) => url, - Err(e) => panic!( - "Failed to read database URL from env var LEMMY_DATABASE_URL: {}", - e - ), +async fn build_db_pool_settings_opt( + settings: Option<&Settings>, +) -> Result { + let db_url = get_database_url(settings); + let pool_size = settings.map(|s| s.database.pool_size).unwrap_or(5); + // We only support TLS with sslmode=require currently + let tls_enabled = db_url.contains("sslmode=require"); + let manager = if tls_enabled { + // diesel-async does not support any TLS connections out of the box, so we need to manually + // provide a setup function which handles creating the connection + AsyncDieselConnectionManager::::new_with_setup(&db_url, establish_connection) + } else { + AsyncDieselConnectionManager::::new(&db_url) }; - let conn = - PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)); - embedded_migrations::run(&conn).expect("load migrations"); - conn + let pool = Pool::builder(manager) + .max_size(pool_size) + .wait_timeout(POOL_TIMEOUT) + .create_timeout(POOL_TIMEOUT) + .recycle_timeout(POOL_TIMEOUT) + .runtime(Runtime::Tokio1) + .build()?; + + // If there's no settings, that means its a unit test, and migrations need to be run + if settings.is_none() { + run_migrations(&db_url); + } + + Ok(pool) +} + +fn establish_connection(config: &str) -> BoxFuture> { + let fut = async { + let rustls_config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(Arc::new(NoCertVerifier {})) + .with_no_client_auth(); + + let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config); + let (client, conn) = tokio_postgres::connect(config, tls) + .await + .map_err(|e| ConnectionError::BadConnection(e.to_string()))?; + tokio::spawn(async move { + if let Err(e) = conn.await { + error!("Database connection failed: {e}"); + } + }); + AsyncPgConnection::try_from(client).await + }; + fut.boxed() +} + +struct NoCertVerifier {} + +impl ServerCertVerifier for NoCertVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: SystemTime, + ) -> Result { + // Will verify all (even invalid) certs without any checks (sslmode=require) + Ok(ServerCertVerified::assertion()) + } +} + +pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); + +pub fn run_migrations(db_url: &str) { + // Needs to be a sync connection + let mut conn = + PgConnection::establish(db_url).unwrap_or_else(|e| panic!("Error connecting to {db_url}: {e}")); + info!("Running Database migrations (This may take a long time)..."); + let _ = &mut conn + .run_pending_migrations(MIGRATIONS) + .unwrap_or_else(|e| panic!("Couldn't run DB Migrations: {e}")); + info!("Database migrations complete."); +} + +pub async fn build_db_pool(settings: &Settings) -> Result { + build_db_pool_settings_opt(Some(settings)).await +} + +pub async fn build_db_pool_for_tests() -> ActualDbPool { + build_db_pool_settings_opt(None) + .await + .expect("db pool missing") +} + +pub fn get_database_url(settings: Option<&Settings>) -> String { + // The env var should override anything in the settings config + match get_database_url_from_env() { + Ok(url) => url, + Err(e) => match settings { + Some(settings) => settings.get_database_url(), + None => panic!("Failed to read database URL from env var LEMMY_DATABASE_URL: {e}"), + }, + } } pub fn naive_now() -> NaiveDateTime { chrono::prelude::Utc::now().naive_utc() } +pub fn post_to_comment_sort_type(sort: SortType) -> CommentSortType { + match sort { + SortType::Active | SortType::Hot => CommentSortType::Hot, + SortType::New | SortType::NewComments | SortType::MostComments => CommentSortType::New, + SortType::Old => CommentSortType::Old, + SortType::TopHour + | SortType::TopSixHour + | SortType::TopTwelveHour + | SortType::TopDay + | SortType::TopAll + | SortType::TopWeek + | SortType::TopYear + | SortType::TopMonth + | SortType::TopThreeMonths + | SortType::TopSixMonths + | SortType::TopNineMonths => CommentSortType::Top, + } +} + static EMAIL_REGEX: Lazy = Lazy::new(|| { Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$") .expect("compile email regex") }); pub mod functions { - use diesel::sql_types::*; + use diesel::sql_types::{BigInt, Text, Timestamp}; sql_function! { fn hot_rank(score: BigInt, time: Timestamp) -> Integer; @@ -100,12 +376,11 @@ pub mod functions { sql_function!(fn lower(x: Text) -> Text); } -impl ToSql for DbUrl -where - String: ToSql, -{ - fn to_sql(&self, out: &mut Output) -> diesel::serialize::Result { - self.0.to_string().to_sql(out) +pub const DELETED_REPLACEMENT_TEXT: &str = "*Permanently Deleted*"; + +impl ToSql for DbUrl { + fn to_sql(&self, out: &mut Output) -> diesel::serialize::Result { + >::to_sql(&self.0.to_string(), &mut out.reborrow()) } } @@ -113,24 +388,27 @@ impl FromSql for DbUrl where String: FromSql, { - fn from_sql(bytes: Option<&DB::RawValue>) -> diesel::deserialize::Result { - let str = String::from_sql(bytes)?; - Ok(DbUrl(Url::parse(&str)?)) + fn from_sql(value: DB::RawValue<'_>) -> diesel::deserialize::Result { + let str = String::from_sql(value)?; + Ok(DbUrl(Box::new(Url::parse(&str)?))) } } impl From> for DbUrl where - Kind: ApubObject + Send + 'static, - for<'de2> ::ApubType: serde::Deserialize<'de2>, + Kind: Object + Send + 'static, + for<'de2> ::Kind: serde::Deserialize<'de2>, { fn from(id: ObjectId) -> Self { - DbUrl(id.into()) + DbUrl(Box::new(id.into())) } } #[cfg(test)] mod tests { + #![allow(clippy::unwrap_used)] + #![allow(clippy::indexing_slicing)] + use super::{fuzzy_search, *}; use crate::utils::is_email_regex; @@ -152,7 +430,7 @@ mod tests { #[test] fn test_diesel_option_overwrite() { assert_eq!(diesel_option_overwrite(&None), None); - assert_eq!(diesel_option_overwrite(&Some("".to_string())), Some(None)); + assert_eq!(diesel_option_overwrite(&Some(String::new())), Some(None)); assert_eq!( diesel_option_overwrite(&Some("test".to_string())), Some(Some("test".to_string())) @@ -163,13 +441,10 @@ mod tests { fn test_diesel_option_overwrite_to_url() { assert!(matches!(diesel_option_overwrite_to_url(&None), Ok(None))); assert!(matches!( - diesel_option_overwrite_to_url(&Some("".to_string())), + diesel_option_overwrite_to_url(&Some(String::new())), Ok(Some(None)) )); - assert!(matches!( - diesel_option_overwrite_to_url(&Some("invalid_url".to_string())), - Err(_) - )); + assert!(diesel_option_overwrite_to_url(&Some("invalid_url".to_string())).is_err()); let example_url = "https://example.com"; assert!(matches!( diesel_option_overwrite_to_url(&Some(example_url.to_string())),