X-Git-Url: http://these/git/?a=blobdiff_plain;f=src%2Flib.rs;h=e07ae2685d478736c756b320de429bd2f1914e5c;hb=5d23ef960e3361d1cc38c1aff1d907d8da58b90f;hp=5d5f00b0de01960d0181212995656c5ca2bfffce;hpb=9d4973829bb2ff670401ab0d8b7810db103df7b9;p=lemmy.git

diff --git a/src/lib.rs b/src/lib.rs
index 5d5f00b0..e07ae268 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,307 +1,268 @@
-#![recursion_limit = "512"]
-#[macro_use]
-extern crate lazy_static;
-extern crate actix;
-extern crate actix_web;
-extern crate base64;
-extern crate bcrypt;
-extern crate captcha;
-extern crate chrono;
-extern crate diesel;
-extern crate dotenv;
-extern crate jsonwebtoken;
-extern crate log;
-extern crate openssl;
-extern crate reqwest;
-extern crate rss;
-extern crate serde;
-extern crate serde_json;
-extern crate sha2;
-extern crate strum;
-
-pub mod api;
-pub mod apub;
+pub mod api_routes_http;
 pub mod code_migrations;
-pub mod request;
-pub mod routes;
-pub mod version;
-pub mod websocket;
-
-use crate::{
-  request::{retry, RecvError},
-  websocket::chat_server::ChatServer,
+#[cfg(feature = "prometheus-metrics")]
+pub mod prometheus_metrics;
+pub mod root_span_builder;
+pub mod scheduled_tasks;
+#[cfg(feature = "console")]
+pub mod telemetry;
+
+use crate::{code_migrations::run_advanced_migrations, root_span_builder::QuieterRootSpanBuilder};
+use activitypub_federation::config::{FederationConfig, FederationMiddleware};
+use actix_cors::Cors;
+use actix_web::{
+  middleware::{self, ErrorHandlers},
+  web::Data,
+  App,
+  HttpServer,
+  Result,
+};
+use lemmy_api_common::{
+  context::LemmyContext,
+  lemmy_db_views::structs::SiteView,
+  request::build_user_agent,
+  send_activity::MATCH_OUTGOING_ACTIVITIES,
+  utils::{
+    check_private_instance_and_federation_enabled,
+    local_site_rate_limit_to_rate_limit_config,
+  },
+};
+use lemmy_apub::{
+  activities::{handle_outgoing_activities, match_outgoing_activities},
+  VerifyUrlData,
+  FEDERATION_HTTP_FETCH_LIMIT,
+};
+use lemmy_db_schema::{
+  source::secret::Secret,
+  utils::{build_db_pool, get_database_url, run_migrations},
+};
+use lemmy_routes::{feeds, images, nodeinfo, webfinger};
+use lemmy_utils::{
+  error::LemmyError,
+  rate_limit::RateLimitCell,
+  response::jsonify_plain_text_errors,
+  settings::SETTINGS,
+  SYNCHRONOUS_FEDERATION,
 };
-use actix::Addr;
-use anyhow::anyhow;
-use background_jobs::QueueHandle;
-use lemmy_db::DbPool;
-use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
-use log::error;
-use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use reqwest::Client;
-use serde::Deserialize;
-use std::process::Command;
-
-pub struct LemmyContext {
-  pub pool: DbPool,
-  pub chat_server: Addr<ChatServer>,
-  pub client: Client,
-  pub activity_queue: QueueHandle,
-}
+use reqwest_middleware::ClientBuilder;
+use reqwest_tracing::TracingMiddleware;
+use std::{env, thread, time::Duration};
+use tracing::subscriber::set_global_default;
+use tracing_actix_web::TracingLogger;
+use tracing_error::ErrorLayer;
+use tracing_log::LogTracer;
+use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry};
+use url::Url;
+#[cfg(feature = "prometheus-metrics")]
+use {
+  actix_web_prom::PrometheusMetricsBuilder,
+  prometheus::default_registry,
+  prometheus_metrics::serve_prometheus,
+};
 
-impl LemmyContext {
-  pub fn new(
-    pool: DbPool,
-    chat_server: Addr<ChatServer>,
-    client: Client,
-    activity_queue: QueueHandle,
-  ) -> LemmyContext {
-    LemmyContext {
-      pool,
-      chat_server,
-      client,
-      activity_queue,
-    }
-  }
-  pub fn pool(&self) -> &DbPool {
-    &self.pool
-  }
-  pub fn chat_server(&self) -> &Addr<ChatServer> {
-    &self.chat_server
-  }
-  pub fn client(&self) -> &Client {
-    &self.client
-  }
-  pub fn activity_queue(&self) -> &QueueHandle {
-    &self.activity_queue
-  }
-}
+/// Max timeout for http requests
+pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
 
-impl Clone for LemmyContext {
-  fn clone(&self) -> Self {
-    LemmyContext::new(
-      self.pool.clone(),
-      self.chat_server.clone(),
-      self.client.clone(),
-      self.activity_queue.clone(),
-    )
-  }
-}
+/// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
+pub async fn start_lemmy_server() -> Result<(), LemmyError> {
+  let args: Vec<String> = env::args().collect();
 
-#[derive(Deserialize, Debug)]
-pub struct IframelyResponse {
-  title: Option<String>,
-  description: Option<String>,
-  thumbnail_url: Option<String>,
-  html: Option<String>,
-}
+  let scheduled_tasks_enabled = args.get(1) != Some(&"--disable-scheduled-tasks".to_string());
 
-pub async fn fetch_iframely(client: &Client, url: &str) -> Result<IframelyResponse, LemmyError> {
-  let fetch_url = format!("http://iframely/oembed?url={}", url);
+  let settings = SETTINGS.to_owned();
 
-  let response = retry(|| client.get(&fetch_url).send()).await?;
+  // Run the DB migrations
+  let db_url = get_database_url(Some(&settings));
+  run_migrations(&db_url);
 
-  let res: IframelyResponse = response
-    .json()
-    .await
-    .map_err(|e| RecvError(e.to_string()))?;
-  Ok(res)
-}
+  // Set up the connection pool
+  let pool = build_db_pool(&settings).await?;
 
-#[derive(Deserialize, Debug, Clone)]
-pub struct PictrsResponse {
-  files: Vec<PictrsFile>,
-  msg: String,
-}
+  // Run the Code-required migrations
+  run_advanced_migrations(&mut (&pool).into(), &settings).await?;
 
-#[derive(Deserialize, Debug, Clone)]
-pub struct PictrsFile {
-  file: String,
-  delete_token: String,
-}
+  // Initialize the secrets
+  let secret = Secret::init(&mut (&pool).into())
+    .await
+    .expect("Couldn't initialize secrets.");
 
-pub async fn fetch_pictrs(client: &Client, image_url: &str) -> Result<PictrsResponse, LemmyError> {
-  is_image_content_type(client, image_url).await?;
+  // Make sure the local site is set up.
+  let site_view = SiteView::read_local(&mut (&pool).into())
+    .await
+    .expect("local site not set up");
+  let local_site = site_view.local_site;
+  let federation_enabled = local_site.federation_enabled;
 
-  let fetch_url = format!(
-    "http://pictrs:8080/image/download?url={}",
-    utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
-  );
+  if federation_enabled {
+    println!("federation enabled, host is {}", &settings.hostname);
+  }
 
-  let response = retry(|| client.get(&fetch_url).send()).await?;
+  check_private_instance_and_federation_enabled(&local_site)?;
 
-  let response: PictrsResponse = response
-    .json()
-    .await
-    .map_err(|e| RecvError(e.to_string()))?;
+  // Set up the rate limiter
+  let rate_limit_config =
+    local_site_rate_limit_to_rate_limit_config(&site_view.local_site_rate_limit);
+  let rate_limit_cell = RateLimitCell::new(rate_limit_config).await;
 
-  if response.msg == "ok" {
-    Ok(response)
-  } else {
-    Err(anyhow!("{}", &response.msg).into())
-  }
-}
+  println!(
+    "Starting http server at {}:{}",
+    settings.bind, settings.port
+  );
 
-async fn fetch_iframely_and_pictrs_data(
-  client: &Client,
-  url: Option<String>,
-) -> (
-  Option<String>,
-  Option<String>,
-  Option<String>,
-  Option<String>,
-) {
-  match &url {
-    Some(url) => {
-      // Fetch iframely data
-      let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
-        match fetch_iframely(client, url).await {
-          Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
-          Err(e) => {
-            error!("iframely err: {}", e);
-            (None, None, None, None)
-          }
-        };
-
-      // Fetch pictrs thumbnail
-      let pictrs_hash = match iframely_thumbnail_url {
-        Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
-          Ok(res) => Some(res.files[0].file.to_owned()),
-          Err(e) => {
-            error!("pictrs err: {}", e);
-            None
-          }
-        },
-        // Try to generate a small thumbnail if iframely is not supported
-        None => match fetch_pictrs(client, &url).await {
-          Ok(res) => Some(res.files[0].file.to_owned()),
-          Err(e) => {
-            error!("pictrs err: {}", e);
-            None
-          }
-        },
-      };
-
-      // The full urls are necessary for federation
-      let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
-        Some(format!(
-          "{}://{}/pictrs/image/{}",
-          get_apub_protocol_string(),
-          Settings::get().hostname,
-          pictrs_hash
-        ))
-      } else {
-        None
-      };
-
-      (
-        iframely_title,
-        iframely_description,
-        iframely_html,
-        pictrs_thumbnail,
-      )
-    }
-    None => (None, None, None, None),
-  }
-}
+  let user_agent = build_user_agent(&settings);
+  let reqwest_client = Client::builder()
+    .user_agent(user_agent.clone())
+    .timeout(REQWEST_TIMEOUT)
+    .connect_timeout(REQWEST_TIMEOUT)
+    .build()?;
+
+  let client = ClientBuilder::new(reqwest_client.clone())
+    .with(TracingMiddleware::default())
+    .build();
+
+  // Pictrs cannot use the retry middleware
+  let pictrs_client = ClientBuilder::new(reqwest_client.clone())
+    .with(TracingMiddleware::default())
+    .build();
+
+  let context = LemmyContext::create(
+    pool.clone(),
+    client.clone(),
+    secret.clone(),
+    rate_limit_cell.clone(),
+  );
 
-pub async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
-  let response = retry(|| client.get(test).send()).await?;
-
-  if response
-    .headers()
-    .get("Content-Type")
-    .ok_or_else(|| anyhow!("No Content-Type header"))?
-    .to_str()?
-    .starts_with("image/")
-  {
-    Ok(())
-  } else {
-    Err(anyhow!("Not an image type.").into())
+  if scheduled_tasks_enabled {
+    // Schedules various cleanup tasks for the DB
+    thread::spawn({
+      let context = context.clone();
+      move || {
+        scheduled_tasks::setup(db_url, user_agent, context)
+          .expect("Couldn't set up scheduled_tasks");
+      }
+    });
   }
-}
 
-pub fn captcha_espeak_wav_base64(captcha: &str) -> Result<String, LemmyError> {
-  let mut built_text = String::new();
-
-  // Building proper speech text for espeak
-  for mut c in captcha.chars() {
-    let new_str = if c.is_alphabetic() {
-      if c.is_lowercase() {
-        c.make_ascii_uppercase();
-        format!("lower case {} ... ", c)
-      } else {
-        c.make_ascii_uppercase();
-        format!("capital {} ... ", c)
-      }
-    } else {
-      format!("{} ...", c)
+  #[cfg(feature = "prometheus-metrics")]
+  serve_prometheus(settings.prometheus.as_ref(), context.clone());
+
+  let settings_bind = settings.clone();
+
+  let federation_config = FederationConfig::builder()
+    .domain(settings.hostname.clone())
+    .app_data(context.clone())
+    .client(client.clone())
+    .http_fetch_limit(FEDERATION_HTTP_FETCH_LIMIT)
+    .worker_count(settings.worker_count)
+    .retry_count(settings.retry_count)
+    .debug(*SYNCHRONOUS_FEDERATION)
+    .http_signature_compat(true)
+    .url_verifier(Box::new(VerifyUrlData(context.inner_pool().clone())))
+    .build()
+    .await?;
+
+  // this must come before the HttpServer creation
+  // creates a middleware that populates http metrics for each path, method, and status code
+  #[cfg(feature = "prometheus-metrics")]
+  let prom_api_metrics = PrometheusMetricsBuilder::new("lemmy_api")
+    .registry(default_registry().clone())
+    .build()
+    .expect("Should always be buildable");
+
+  MATCH_OUTGOING_ACTIVITIES
+    .set(Box::new(move |d, c| {
+      Box::pin(match_outgoing_activities(d, c))
+    }))
+    .expect("set function pointer");
+  let request_data = federation_config.to_request_data();
+  let outgoing_activities_task = tokio::task::spawn(handle_outgoing_activities(request_data));
+
+  // Create Http server with websocket support
+  HttpServer::new(move || {
+    let cors_origin = env::var("LEMMY_CORS_ORIGIN");
+    let cors_config = match (cors_origin, cfg!(debug_assertions)) {
+      (Ok(origin), false) => Cors::default()
+        .allowed_origin(&origin)
+        .allowed_origin(&settings.get_protocol_and_hostname()),
+      _ => Cors::default()
+        .allow_any_origin()
+        .allow_any_method()
+        .allow_any_header()
+        .expose_any_header()
+        .max_age(3600),
     };
 
-    built_text.push_str(&new_str);
-  }
-
-  espeak_wav_base64(&built_text)
+    let app = App::new()
+      .wrap(middleware::Logger::new(
+        // This is the default log format save for the usage of %{r}a over %a to guarantee to record the client's (forwarded) IP and not the last peer address, since the latter is frequently just a reverse proxy
+        "%{r}a '%r' %s %b '%{Referer}i' '%{User-Agent}i' %T",
+      ))
+      .wrap(middleware::Compress::default())
+      .wrap(cors_config)
+      .wrap(TracingLogger::<QuieterRootSpanBuilder>::new())
+      .wrap(ErrorHandlers::new().default_handler(jsonify_plain_text_errors))
+      .app_data(Data::new(context.clone()))
+      .app_data(Data::new(rate_limit_cell.clone()))
+      .wrap(FederationMiddleware::new(federation_config.clone()));
+
+    #[cfg(feature = "prometheus-metrics")]
+    let app = app.wrap(prom_api_metrics.clone());
+
+    // The routes
+    app
+      .configure(|cfg| api_routes_http::config(cfg, rate_limit_cell))
+      .configure(|cfg| {
+        if federation_enabled {
+          lemmy_apub::http::routes::config(cfg);
+          webfinger::config(cfg);
+        }
+      })
+      .configure(feeds::config)
+      .configure(|cfg| images::config(cfg, pictrs_client.clone(), rate_limit_cell))
+      .configure(nodeinfo::config)
+  })
+  .bind((settings_bind.bind, settings_bind.port))?
+  .run()
+  .await?;
+
+  // Wait for outgoing apub sends to complete
+  outgoing_activities_task.await??;
+
+  Ok(())
 }
 
-pub fn espeak_wav_base64(text: &str) -> Result<String, LemmyError> {
-  // Make a temp file path
-  let uuid = uuid::Uuid::new_v4().to_string();
-  let file_path = format!("/tmp/lemmy_espeak_{}.wav", &uuid);
+pub fn init_logging(opentelemetry_url: &Option<Url>) -> Result<(), LemmyError> {
+  LogTracer::init()?;
 
-  // Write the wav file
-  Command::new("espeak")
-    .arg("-w")
-    .arg(&file_path)
-    .arg(text)
-    .status()?;
+  let log_description = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into());
 
-  // Read the wav file bytes
-  let bytes = std::fs::read(&file_path)?;
+  let targets = log_description
+    .trim()
+    .trim_matches('"')
+    .parse::<Targets>()?;
 
-  // Delete the file
-  std::fs::remove_file(file_path)?;
+  let format_layer = {
+    #[cfg(feature = "json-log")]
+    let layer = tracing_subscriber::fmt::layer().json();
+    #[cfg(not(feature = "json-log"))]
+    let layer = tracing_subscriber::fmt::layer();
 
-  // Convert to base64
-  let base64 = base64::encode(bytes);
+    layer.with_filter(targets.clone())
+  };
 
-  Ok(base64)
-}
+  let subscriber = Registry::default()
+    .with(format_layer)
+    .with(ErrorLayer::default());
 
-#[cfg(test)]
-mod tests {
-  use crate::{captcha_espeak_wav_base64, is_image_content_type};
-
-  #[test]
-  fn test_image() {
-    actix_rt::System::new("tset_image").block_on(async move {
-      let client = reqwest::Client::default();
-      assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
-      assert!(is_image_content_type(&client,
-                                    "https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
-      )
-        .await.is_err()
-      );
-    });
-  }
-
-  #[test]
-  fn test_espeak() {
-    assert!(captcha_espeak_wav_base64("WxRt2l").is_ok())
+  if let Some(_url) = opentelemetry_url {
+    #[cfg(feature = "console")]
+    telemetry::init_tracing(_url.as_ref(), subscriber, targets)?;
+    #[cfg(not(feature = "console"))]
+    tracing::error!("Feature `console` must be enabled for opentelemetry tracing");
+  } else {
+    set_global_default(subscriber)?;
   }
 
-  // These helped with testing
-  // #[test]
-  // fn test_iframely() {
-  //   let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
-  //   assert!(res.is_ok());
-  // }
-
-  // #[test]
-  // fn test_pictshare() {
-  //   let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
-  //   assert!(res.is_ok());
-  //   let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
-  //   assert!(res_other.is_err());
-  // }
+  Ok(())
 }