Remove `actix_rt` & use standard tokio spawn (#3158)
authorcetra3 <cetra3@hotmail.com>
Mon, 26 Jun 2023 08:24:11 +0000 (17:54 +0930)
committerGitHub <noreply@github.com>
Mon, 26 Jun 2023 08:24:11 +0000 (10:24 +0200)
* Remove `actix_rt` & use standard tokio spawn

* Adjust rust log back down

* Format correctly

* Update cargo lock

* Add DB settings

* Change name and update to latest rev

* Clean up formatting changes

* Move `worker_count` and `worker_retry_count` to settings

* Update defaults

* Use `0.4.4` instead of git branch

23 files changed:
Cargo.lock
Cargo.toml
config/defaults.hjson
crates/api_common/Cargo.toml
crates/api_common/src/request.rs
crates/api_common/src/site.rs
crates/api_crud/src/site/create.rs
crates/api_crud/src/site/update.rs
crates/apub/Cargo.toml
crates/apub/src/collections/community_moderators.rs
crates/apub/src/objects/comment.rs
crates/apub/src/objects/community.rs
crates/apub/src/objects/instance.rs
crates/apub/src/objects/person.rs
crates/apub/src/objects/post.rs
crates/apub/src/objects/private_message.rs
crates/db_schema/src/schema.rs
crates/db_schema/src/source/local_site.rs
crates/utils/src/settings/structs.rs
migrations/2023-06-19-055530_add_retry_worker_setting/down.sql [new file with mode: 0644]
migrations/2023-06-19-055530_add_retry_worker_setting/up.sql [new file with mode: 0644]
src/lib.rs
src/main.rs

index 9590e8f1399324aec7289c298bd2358d16b0c5a9..9d575f5782f9064e4adb1dfb5efdf14e039357d9 100644 (file)
@@ -2543,7 +2543,6 @@ dependencies = [
 name = "lemmy_api_common"
 version = "0.18.0"
 dependencies = [
- "actix-rt",
  "actix-web",
  "anyhow",
  "chrono",
@@ -2561,6 +2560,7 @@ dependencies = [
  "rosetta-i18n",
  "serde",
  "serde_with",
+ "tokio",
  "tracing",
  "ts-rs",
  "url",
@@ -2592,7 +2592,6 @@ name = "lemmy_apub"
 version = "0.18.0"
 dependencies = [
  "activitypub_federation",
- "actix-rt",
  "actix-web",
  "anyhow",
  "assert-json-diff",
@@ -2620,6 +2619,7 @@ dependencies = [
  "sha2",
  "strum_macros",
  "task-local-extensions",
+ "tokio",
  "tracing",
  "url",
  "uuid",
index 430deb082c41dd11c6b675875ade2965106a1e9e..07e41ab3b896fe56372663ebc148a148e1cdcc04 100644 (file)
@@ -89,7 +89,7 @@ anyhow = "1.0.71"
 diesel_ltree = "0.3.0"
 typed-builder = "0.10.0"
 serial_test = "0.9.0"
-tokio = "1.28.2"
+tokio = { version = "1.28.2", features = ["full"] }
 sha2 = "0.10.6"
 regex = "1.8.4"
 once_cell = "1.18.0"
index 4c38ddd45547fbd9eccaf1858286a837b071b221..6032f8fc9ae4520be514fa689e43e0f97eaec9a2 100644 (file)
@@ -76,4 +76,8 @@
   port: 8536
   # Whether the site is available over TLS. Needs to be true for federation to work.
   tls_enabled: true
+  # The number of activitypub federation workers that can be in-flight concurrently
+  worker_count: 0
+  # The number of activitypub federation retry workers that can be in-flight concurrently
+  retry_count: 0
 }
index 46045d8057a6c849beb00387094eed71dcb163ba..339d233a16b1ba553f3d69854a2d8c515ae9a628 100644 (file)
@@ -38,7 +38,7 @@ encoding = { version = "0.2.33", optional = true }
 anyhow = { workspace = true }
 futures = { workspace = true }
 uuid = { workspace = true }
-actix-rt = { workspace = true }
+tokio = { workspace = true }
 reqwest = { workspace = true }
 ts-rs = { workspace = true, optional = true }
 actix-web = { workspace = true }
index c6f71b868a3ff000621b0b75e568f0f872e4640e..3139193a6a8991c4128db4502e843bd4536fc701 100644 (file)
@@ -271,7 +271,7 @@ mod tests {
   use url::Url;
 
   // These helped with testing
-  #[actix_rt::test]
+  #[tokio::test]
   async fn test_site_metadata() {
     let settings = &SETTINGS.clone();
     let client = reqwest::Client::builder()
index 4d488ec1b812befc6bf23bf68bc21cf2b3df2063..865acc0dc2f81bb84717bd82fc82c255b20f5e1a 100644 (file)
@@ -177,7 +177,6 @@ pub struct CreateSite {
   pub rate_limit_search_per_second: Option<i32>,
   pub federation_enabled: Option<bool>,
   pub federation_debug: Option<bool>,
-  pub federation_worker_count: Option<i32>,
   pub captcha_enabled: Option<bool>,
   pub captcha_difficulty: Option<String>,
   pub allowed_instances: Option<Vec<String>>,
@@ -250,8 +249,6 @@ pub struct EditSite {
   pub federation_enabled: Option<bool>,
   /// Enables federation debugging.
   pub federation_debug: Option<bool>,
-  /// The number of federation workers.
-  pub federation_worker_count: Option<i32>,
   /// Whether to enable captchas for signups.
   pub captcha_enabled: Option<bool>,
   /// The captcha difficulty. Can be easy, medium, or hard
index 2a51309a42d8c76bc94c0f1ea036faff9e8c0e5c..a1669baef0341569f1590fbb810a2ad6c85c37af 100644 (file)
@@ -122,7 +122,6 @@ impl PerformCrud for CreateSite {
       .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex))
       .actor_name_max_length(data.actor_name_max_length)
       .federation_enabled(data.federation_enabled)
-      .federation_worker_count(data.federation_worker_count)
       .captcha_enabled(data.captcha_enabled)
       .captcha_difficulty(data.captcha_difficulty.clone())
       .build();
index fadde0a0bb63f982431260f8f0d6fea103fd8ae2..6664d549a4f464b96536975e7c4d7919e3bff351 100644 (file)
@@ -123,7 +123,6 @@ impl PerformCrud for EditSite {
       .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex))
       .actor_name_max_length(data.actor_name_max_length)
       .federation_enabled(data.federation_enabled)
-      .federation_worker_count(data.federation_worker_count)
       .captcha_enabled(data.captcha_enabled)
       .captcha_difficulty(data.captcha_difficulty.clone())
       .reports_email_admins(data.reports_email_admins)
index 2007b541a441cd3808968fdd6e244d6703449240..8570541f740503add6c3fd0c535295d5f5c85a9a 100644 (file)
@@ -25,7 +25,7 @@ chrono = { workspace = true }
 serde_json = { workspace = true }
 serde = { workspace = true }
 actix-web = { workspace = true }
-actix-rt = { workspace = true }
+tokio = {workspace = true}
 tracing = { workspace = true }
 strum_macros = { workspace = true }
 url = { workspace = true }
index c439da710d23c0ee4d32c8f1a98b45bf44e871a6..d53f86280d950dd2a5504cc7f00fb862a85b2ba4 100644 (file)
@@ -120,7 +120,7 @@ mod tests {
   };
   use serial_test::serial;
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_lemmy_community_moderators() {
     let context = init_context().await;
index e2a03b8b3f8b69f62be3c8b1fe1ff25ba8f4d106..16cb1542b961dd91261c5b81dbf848417017a2c5 100644 (file)
@@ -223,7 +223,7 @@ pub(crate) mod tests {
     LocalSite::delete(context.pool()).await.unwrap();
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   pub(crate) async fn test_parse_lemmy_comment() {
     let context = init_context().await;
@@ -249,7 +249,7 @@ pub(crate) mod tests {
     cleanup(data, &context).await;
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_pleroma_comment() {
     let context = init_context().await;
@@ -279,7 +279,7 @@ pub(crate) mod tests {
     cleanup(data, &context).await;
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_html_to_markdown_sanitize() {
     let parsed = parse_html("<script></script><b>hello</b>");
index 6526d2d26eff08f05b9dad3bf3930733bf593714..888a7f4581b60c7d7af51ee0d10039ace140a0ec 100644 (file)
@@ -242,7 +242,7 @@ pub(crate) mod tests {
     community
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_lemmy_community() {
     let context = init_context().await;
index 72d133441bfe3e67e7864d0fe0928c9059fc96b3..6cd27fbbdacdfbb3555c056d1165593ba14265ba 100644 (file)
@@ -221,7 +221,7 @@ pub(crate) mod tests {
     site
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_lemmy_instance() {
     let context = init_context().await;
index c71d46ccff80bd0c13ccde672b730a8a0ce9def1..3eeb733fd71ec80e982c00581e1afb62c47dfa53 100644 (file)
@@ -223,7 +223,7 @@ pub(crate) mod tests {
     (person, site)
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_lemmy_person() {
     let context = init_context().await;
@@ -236,7 +236,7 @@ pub(crate) mod tests {
     cleanup((person, site), &context).await;
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_pleroma_person() {
     let context = init_context().await;
index b255ffb9b66efe93b7cd968a12b39fe1d486f395..4ef9351ab474fffd073a682b88f4ba725a25c184 100644 (file)
@@ -281,7 +281,7 @@ mod tests {
   use lemmy_db_schema::source::site::Site;
   use serial_test::serial;
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_lemmy_post() {
     let context = init_context().await;
index 01f576ff8ff667f7ed8679850271ad9cc9a8d4f1..ae2637c58e9a40aaabe52972e530f0851ed452db 100644 (file)
@@ -187,7 +187,7 @@ mod tests {
     Site::delete(context.pool(), data.2.id).await.unwrap();
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_lemmy_pm() {
     let context = init_context().await;
@@ -213,7 +213,7 @@ mod tests {
     cleanup(data, &context).await;
   }
 
-  #[actix_rt::test]
+  #[tokio::test]
   #[serial]
   async fn test_parse_pleroma_pm() {
     let context = init_context().await;
index ac4ddc47a4277b968b703c13f649fbe9541668c2..6714913f468b357379ab6c10ea78b811e7235113 100644 (file)
@@ -339,7 +339,6 @@ diesel::table! {
         slur_filter_regex -> Nullable<Text>,
         actor_name_max_length -> Int4,
         federation_enabled -> Bool,
-        federation_worker_count -> Int4,
         captcha_enabled -> Bool,
         #[max_length = 255]
         captcha_difficulty -> Varchar,
index e65a615358a29ac497959f224ac4cee32de54c94..be93717a9840bb9bcb5901ee5ddb047052481188 100644 (file)
@@ -50,8 +50,6 @@ pub struct LocalSite {
   pub actor_name_max_length: i32,
   /// Whether federation is enabled.
   pub federation_enabled: bool,
-  /// The number of concurrent federation http workers.
-  pub federation_worker_count: i32,
   /// Whether captcha is enabled.
   pub captcha_enabled: bool,
   /// The captcha difficulty.
@@ -85,7 +83,6 @@ pub struct LocalSiteInsertForm {
   pub slur_filter_regex: Option<String>,
   pub actor_name_max_length: Option<i32>,
   pub federation_enabled: Option<bool>,
-  pub federation_worker_count: Option<i32>,
   pub captcha_enabled: Option<bool>,
   pub captcha_difficulty: Option<String>,
   pub registration_mode: Option<RegistrationMode>,
@@ -112,7 +109,6 @@ pub struct LocalSiteUpdateForm {
   pub slur_filter_regex: Option<Option<String>>,
   pub actor_name_max_length: Option<i32>,
   pub federation_enabled: Option<bool>,
-  pub federation_worker_count: Option<i32>,
   pub captcha_enabled: Option<bool>,
   pub captcha_difficulty: Option<String>,
   pub registration_mode: Option<RegistrationMode>,
index 6e200b224dcac4514f36a5c486081caab7b51583..5d0e642f6a2a55787ebfe7c34e2b5a20c67f8207 100644 (file)
@@ -39,6 +39,12 @@ pub struct Settings {
   #[default(None)]
   #[doku(skip)]
   pub opentelemetry_url: Option<Url>,
+  /// The number of activitypub federation workers that can be in-flight concurrently
+  #[default(0)]
+  pub worker_count: usize,
+  /// The number of activitypub federation retry workers that can be in-flight concurrently
+  #[default(0)]
+  pub retry_count: usize,
 }
 
 #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)]
diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql
new file mode 100644 (file)
index 0000000..e3c200a
--- /dev/null
@@ -0,0 +1 @@
+alter table local_site add column federation_worker_count int default 64 not null;
\ No newline at end of file
diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql
new file mode 100644 (file)
index 0000000..2aac86f
--- /dev/null
@@ -0,0 +1 @@
+alter table local_site drop column federation_worker_count;
\ No newline at end of file
index 86cf400b64f103a38ae7d753bcac556e02a66dd7..d919acc057aa29b042e2fbfdbc0dacb8af78a9d4 100644 (file)
@@ -139,24 +139,23 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
     });
   }
 
+  let settings_bind = settings.clone();
+
   let federation_config = FederationConfig::builder()
     .domain(settings.hostname.clone())
     .app_data(context.clone())
     .client(client.clone())
     .http_fetch_limit(FEDERATION_HTTP_FETCH_LIMIT)
-    .worker_count(local_site.federation_worker_count as usize)
+    .worker_count(settings.worker_count)
+    .retry_count(settings.retry_count)
     .debug(cfg!(debug_assertions))
     .http_signature_compat(true)
     .url_verifier(Box::new(VerifyUrlData(context.pool().clone())))
     .build()
-    .await
-    .expect("configure federation");
+    .await?;
 
   // Create Http server with websocket support
-  let settings_bind = settings.clone();
   HttpServer::new(move || {
-    let context = context.clone();
-
     let cors_config = if cfg!(debug_assertions) {
       Cors::permissive()
     } else {
@@ -173,7 +172,7 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
       ))
       .wrap(cors_config)
       .wrap(TracingLogger::<QuieterRootSpanBuilder>::new())
-      .app_data(Data::new(context))
+      .app_data(Data::new(context.clone()))
       .app_data(Data::new(rate_limit_cell.clone()))
       .wrap(FederationMiddleware::new(federation_config.clone()))
       // The routes
index 315fe84be731dd65940ef8d08728c3f716a6b12f..5fc03ed025d28c567878dad7b8882497f618906a 100644 (file)
@@ -1,7 +1,7 @@
 use lemmy_server::{init_logging, start_lemmy_server};
 use lemmy_utils::{error::LemmyError, settings::SETTINGS};
 
-#[actix_web::main]
+#[tokio::main]
 pub async fn main() -> Result<(), LemmyError> {
   init_logging(&SETTINGS.opentelemetry_url)?;
   #[cfg(not(feature = "embed-pictrs"))]