-#![recursion_limit = "512"]
-#[macro_use]
-extern crate lazy_static;
-extern crate actix;
-extern crate actix_web;
-extern crate base64;
-extern crate bcrypt;
-extern crate captcha;
-extern crate chrono;
-extern crate diesel;
-extern crate dotenv;
-extern crate jsonwebtoken;
-extern crate log;
-extern crate openssl;
-extern crate reqwest;
-extern crate rss;
-extern crate serde;
-extern crate serde_json;
-extern crate sha2;
-extern crate strum;
-
-pub mod api;
-pub mod apub;
+pub mod api_routes_http;
+pub mod api_routes_websocket;
pub mod code_migrations;
-pub mod request;
-pub mod routes;
-pub mod version;
-pub mod websocket;
-
-use crate::{
- request::{retry, RecvError},
+pub mod root_span_builder;
+pub mod scheduled_tasks;
+#[cfg(feature = "console")]
+pub mod telemetry;
+
+use crate::{code_migrations::run_advanced_migrations, root_span_builder::QuieterRootSpanBuilder};
+use actix_web::{middleware, web::Data, App, HttpServer, Result};
+use doku::json::{AutoComments, CommentsStyle, Formatting, ObjectsStyle};
+use lemmy_api_common::{
+ context::LemmyContext,
+ lemmy_db_views::structs::SiteView,
+ request::build_user_agent,
+ utils::{
+ check_private_instance_and_federation_enabled,
+ local_site_rate_limit_to_rate_limit_config,
+ },
websocket::chat_server::ChatServer,
};
-use actix::Addr;
-use anyhow::anyhow;
-use background_jobs::QueueHandle;
-use lemmy_db::DbPool;
-use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
-use log::error;
-use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
+use lemmy_db_schema::{
+ source::secret::Secret,
+ utils::{build_db_pool, get_database_url, run_migrations},
+};
+use lemmy_routes::{feeds, images, nodeinfo, webfinger};
+use lemmy_utils::{
+ error::LemmyError,
+ rate_limit::RateLimitCell,
+ settings::{structs::Settings, SETTINGS},
+};
use reqwest::Client;
-use serde::Deserialize;
-use std::process::Command;
-
-pub struct LemmyContext {
- pub pool: DbPool,
- pub chat_server: Addr<ChatServer>,
- pub client: Client,
- pub activity_queue: QueueHandle,
-}
-
-impl LemmyContext {
- pub fn new(
- pool: DbPool,
- chat_server: Addr<ChatServer>,
- client: Client,
- activity_queue: QueueHandle,
- ) -> LemmyContext {
- LemmyContext {
- pool,
- chat_server,
- client,
- activity_queue,
- }
- }
- pub fn pool(&self) -> &DbPool {
- &self.pool
- }
- pub fn chat_server(&self) -> &Addr<ChatServer> {
- &self.chat_server
- }
- pub fn client(&self) -> &Client {
- &self.client
- }
- pub fn activity_queue(&self) -> &QueueHandle {
- &self.activity_queue
- }
-}
-
-impl Clone for LemmyContext {
- fn clone(&self) -> Self {
- LemmyContext::new(
- self.pool.clone(),
- self.chat_server.clone(),
- self.client.clone(),
- self.activity_queue.clone(),
- )
+use reqwest_middleware::ClientBuilder;
+use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
+use reqwest_tracing::TracingMiddleware;
+use std::{env, sync::Arc, thread, time::Duration};
+use tracing::subscriber::set_global_default;
+use tracing_actix_web::TracingLogger;
+use tracing_error::ErrorLayer;
+use tracing_log::LogTracer;
+use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry};
+use url::Url;
+
+/// Max timeout for http requests
+pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
+
+/// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
+pub async fn start_lemmy_server() -> Result<(), LemmyError> {
+ let args: Vec<String> = env::args().collect();
+ if args.len() == 2 && args[1] == "--print-config-docs" {
+ let fmt = Formatting {
+ auto_comments: AutoComments::none(),
+ comments_style: CommentsStyle {
+ separator: "#".to_owned(),
+ },
+ objects_style: ObjectsStyle {
+ surround_keys_with_quotes: false,
+ use_comma_as_separator: false,
+ },
+ ..Default::default()
+ };
+ println!("{}", doku::to_json_fmt_val(&fmt, &Settings::default()));
+ return Ok(());
}
-}
-#[derive(Deserialize, Debug)]
-pub struct IframelyResponse {
- title: Option<String>,
- description: Option<String>,
- thumbnail_url: Option<String>,
- html: Option<String>,
-}
+ let settings = SETTINGS.to_owned();
-pub async fn fetch_iframely(client: &Client, url: &str) -> Result<IframelyResponse, LemmyError> {
- let fetch_url = format!("http://iframely/oembed?url={}", url);
+ // Set up the bb8 connection pool
+ let db_url = get_database_url(Some(&settings));
+ run_migrations(&db_url);
- let response = retry(|| client.get(&fetch_url).send()).await?;
+ // Run the migrations from code
+ let pool = build_db_pool(&settings).await?;
+ run_advanced_migrations(&pool, &settings).await?;
- let res: IframelyResponse = response
- .json()
+ // Initialize the secrets
+ let secret = Secret::init(&pool)
.await
- .map_err(|e| RecvError(e.to_string()))?;
- Ok(res)
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct PictrsResponse {
- files: Vec<PictrsFile>,
- msg: String,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-pub struct PictrsFile {
- file: String,
- delete_token: String,
-}
-
-pub async fn fetch_pictrs(client: &Client, image_url: &str) -> Result<PictrsResponse, LemmyError> {
- is_image_content_type(client, image_url).await?;
-
- let fetch_url = format!(
- "http://pictrs:8080/image/download?url={}",
- utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
- );
+ .expect("Couldn't initialize secrets.");
- let response = retry(|| client.get(&fetch_url).send()).await?;
-
- let response: PictrsResponse = response
- .json()
+ // Make sure the local site is set up.
+ let site_view = SiteView::read_local(&pool)
.await
- .map_err(|e| RecvError(e.to_string()))?;
-
- if response.msg == "ok" {
- Ok(response)
- } else {
- Err(anyhow!("{}", &response.msg).into())
- }
-}
-
-async fn fetch_iframely_and_pictrs_data(
- client: &Client,
- url: Option<String>,
-) -> (
- Option<String>,
- Option<String>,
- Option<String>,
- Option<String>,
-) {
- match &url {
- Some(url) => {
- // Fetch iframely data
- let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
- match fetch_iframely(client, url).await {
- Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
- Err(e) => {
- error!("iframely err: {}", e);
- (None, None, None, None)
- }
- };
-
- // Fetch pictrs thumbnail
- let pictrs_hash = match iframely_thumbnail_url {
- Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
- Ok(res) => Some(res.files[0].file.to_owned()),
- Err(e) => {
- error!("pictrs err: {}", e);
- None
- }
- },
- // Try to generate a small thumbnail if iframely is not supported
- None => match fetch_pictrs(client, &url).await {
- Ok(res) => Some(res.files[0].file.to_owned()),
- Err(e) => {
- error!("pictrs err: {}", e);
- None
- }
- },
- };
-
- // The full urls are necessary for federation
- let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
- Some(format!(
- "{}://{}/pictrs/image/{}",
- get_apub_protocol_string(),
- Settings::get().hostname,
- pictrs_hash
- ))
- } else {
- None
- };
-
- (
- iframely_title,
- iframely_description,
- iframely_html,
- pictrs_thumbnail,
- )
- }
- None => (None, None, None, None),
- }
-}
-
-pub async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
- let response = retry(|| client.get(test).send()).await?;
+ .expect("local site not set up");
+ let local_site = site_view.local_site;
+ let federation_enabled = local_site.federation_enabled;
- if response
- .headers()
- .get("Content-Type")
- .ok_or_else(|| anyhow!("No Content-Type header"))?
- .to_str()?
- .starts_with("image/")
- {
- Ok(())
- } else {
- Err(anyhow!("Not an image type.").into())
+ if federation_enabled {
+ println!("federation enabled, host is {}", &settings.hostname);
}
-}
-pub fn captcha_espeak_wav_base64(captcha: &str) -> Result<String, LemmyError> {
- let mut built_text = String::new();
+ check_private_instance_and_federation_enabled(&local_site)?;
- // Building proper speech text for espeak
- for mut c in captcha.chars() {
- let new_str = if c.is_alphabetic() {
- if c.is_lowercase() {
- c.make_ascii_uppercase();
- format!("lower case {} ... ", c)
- } else {
- c.make_ascii_uppercase();
- format!("capital {} ... ", c)
- }
- } else {
- format!("{} ...", c)
- };
+ // Set up the rate limiter
+ let rate_limit_config =
+ local_site_rate_limit_to_rate_limit_config(&site_view.local_site_rate_limit);
+ let rate_limit_cell = RateLimitCell::new(rate_limit_config).await;
- built_text.push_str(&new_str);
- }
+ println!(
+ "Starting http server at {}:{}",
+ settings.bind, settings.port
+ );
- espeak_wav_base64(&built_text)
+ let user_agent = build_user_agent(&settings);
+ let reqwest_client = Client::builder()
+ .user_agent(user_agent.clone())
+ .timeout(REQWEST_TIMEOUT)
+ .build()?;
+
+ let retry_policy = ExponentialBackoff {
+ max_n_retries: 3,
+ max_retry_interval: REQWEST_TIMEOUT,
+ min_retry_interval: Duration::from_millis(100),
+ backoff_exponent: 2,
+ };
+
+ let client = ClientBuilder::new(reqwest_client.clone())
+ .with(TracingMiddleware::default())
+ .with(RetryTransientMiddleware::new_with_policy(retry_policy))
+ .build();
+
+ // Pictrs cannot use the retry middleware
+ let pictrs_client = ClientBuilder::new(reqwest_client.clone())
+ .with(TracingMiddleware::default())
+ .build();
+
+ // Schedules various cleanup tasks for the DB
+ thread::spawn(move || {
+ scheduled_tasks::setup(db_url, user_agent).expect("Couldn't set up scheduled_tasks");
+ });
+
+ let chat_server = Arc::new(ChatServer::startup());
+
+ // Create Http server with websocket support
+ let settings_bind = settings.clone();
+ HttpServer::new(move || {
+ let context = LemmyContext::create(
+ pool.clone(),
+ chat_server.clone(),
+ client.clone(),
+ settings.clone(),
+ secret.clone(),
+ rate_limit_cell.clone(),
+ );
+ App::new()
+ .wrap(middleware::Logger::default())
+ .wrap(TracingLogger::<QuieterRootSpanBuilder>::new())
+ .app_data(Data::new(context))
+ .app_data(Data::new(rate_limit_cell.clone()))
+ // The routes
+ .configure(|cfg| api_routes_http::config(cfg, rate_limit_cell))
+ .configure(|cfg| {
+ if federation_enabled {
+ lemmy_apub::http::routes::config(cfg);
+ webfinger::config(cfg);
+ }
+ })
+ .configure(feeds::config)
+ .configure(|cfg| images::config(cfg, pictrs_client.clone(), rate_limit_cell))
+ .configure(nodeinfo::config)
+ })
+ .bind((settings_bind.bind, settings_bind.port))?
+ .run()
+ .await?;
+
+ Ok(())
}
-pub fn espeak_wav_base64(text: &str) -> Result<String, LemmyError> {
- // Make a temp file path
- let uuid = uuid::Uuid::new_v4().to_string();
- let file_path = format!("/tmp/lemmy_espeak_{}.wav", &uuid);
-
- // Write the wav file
- Command::new("espeak")
- .arg("-w")
- .arg(&file_path)
- .arg(text)
- .status()?;
+pub fn init_logging(opentelemetry_url: &Option<Url>) -> Result<(), LemmyError> {
+ LogTracer::init()?;
- // Read the wav file bytes
- let bytes = std::fs::read(&file_path)?;
+ let log_description = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into());
- // Delete the file
- std::fs::remove_file(file_path)?;
+ let targets = log_description
+ .trim()
+ .trim_matches('"')
+ .parse::<Targets>()?;
- // Convert to base64
- let base64 = base64::encode(bytes);
+ let format_layer = tracing_subscriber::fmt::layer().with_filter(targets.clone());
- Ok(base64)
-}
-
-#[cfg(test)]
-mod tests {
- use crate::{captcha_espeak_wav_base64, is_image_content_type};
-
- #[test]
- fn test_image() {
- actix_rt::System::new("tset_image").block_on(async move {
- let client = reqwest::Client::default();
- assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
- assert!(is_image_content_type(&client,
- "https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
- )
- .await.is_err()
- );
- });
- }
+ let subscriber = Registry::default()
+ .with(format_layer)
+ .with(ErrorLayer::default());
- #[test]
- fn test_espeak() {
- assert!(captcha_espeak_wav_base64("WxRt2l").is_ok())
+ if let Some(_url) = opentelemetry_url {
+ #[cfg(feature = "console")]
+ telemetry::init_tracing(_url.as_ref(), subscriber, targets)?;
+ #[cfg(not(feature = "console"))]
+ tracing::error!("Feature `console` must be enabled for opentelemetry tracing");
+ } else {
+ set_global_default(subscriber)?;
}
- // These helped with testing
- // #[test]
- // fn test_iframely() {
- // let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
- // assert!(res.is_ok());
- // }
-
- // #[test]
- // fn test_pictshare() {
- // let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
- // assert!(res.is_ok());
- // let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
- // assert!(res_other.is_err());
- // }
+ Ok(())
}