From: Riley <asonix@asonix.dog>
Date: Thu, 6 Jan 2022 19:10:20 +0000 (-0600)
Subject: Opentelemetry (#1992)
X-Git-Url: http://these/git/%22https:/join-lemmy.org/%7BpictshareAvatarThumbnail%28node.comment.creator_avatar%29%7D?a=commitdiff_plain;h=3032a2745b8e989323ea689b8eb2e38339047e4d;p=lemmy.git

Opentelemetry (#1992)

* Add otlp export

* Add jaeger to dev docker

* More instrument

* Update deps

* Merge from main.

* Add comment about viewing traces

* Fixing reqwest-tracing dep.

* Update pre-commit regex

* Move imports to top

Co-authored-by: Dessalines <tyhou13@gmx.com>
Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
---

diff --git a/Cargo.lock b/Cargo.lock
index 57e1dd74..923399f4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -348,6 +348,27 @@ dependencies = [
  "event-listener",
 ]
 
+[[package]]
+name = "async-stream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
+dependencies = [
+ "proc-macro2 1.0.33",
+ "quote 1.0.10",
+ "syn 1.0.82",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.52"
@@ -1088,6 +1109,12 @@ version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "31586bda1b136406162e381a3185a506cdfc1631708dd40cba2f6628d8634499"
 
+[[package]]
+name = "fixedbitset"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
+
 [[package]]
 name = "flate2"
 version = "1.0.22"
@@ -1491,6 +1518,18 @@ dependencies = [
  "want",
 ]
 
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.5.0"
@@ -1949,6 +1988,8 @@ dependencies = [
  "lemmy_utils",
  "lemmy_websocket",
  "openssl",
+ "opentelemetry",
+ "opentelemetry-otlp",
  "reqwest",
  "reqwest-middleware",
  "reqwest-tracing",
@@ -1960,6 +2001,7 @@ dependencies = [
  "tracing-actix-web",
  "tracing-error",
  "tracing-log",
+ "tracing-opentelemetry",
  "tracing-subscriber",
  "url",
 ]
@@ -2018,6 +2060,7 @@ dependencies = [
  "lemmy_db_views",
  "lemmy_db_views_actor",
  "lemmy_utils",
+ "opentelemetry",
  "rand 0.8.4",
  "reqwest",
  "reqwest-middleware",
@@ -2027,6 +2070,7 @@ dependencies = [
  "strum_macros",
  "tokio",
  "tracing",
+ "tracing-opentelemetry",
 ]
 
 [[package]]
@@ -2287,6 +2331,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "multimap"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+
 [[package]]
 name = "native-tls"
 version = "0.2.8"
@@ -2463,6 +2513,42 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "opentelemetry"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1cf9b1c4e9a6c4de793c632496fa490bdc0e1eea73f0c91394f7b6990935d22"
+dependencies = [
+ "async-trait",
+ "crossbeam-channel",
+ "futures",
+ "js-sys",
+ "lazy_static",
+ "percent-encoding",
+ "pin-project",
+ "rand 0.8.4",
+ "thiserror",
+ "tokio",
+ "tokio-stream",
+]
+
+[[package]]
+name = "opentelemetry-otlp"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f19d4b43842433c420c548c985d158f5628bba5b518e0be64627926d19889992"
+dependencies = [
+ "async-trait",
+ "futures",
+ "http",
+ "opentelemetry",
+ "prost",
+ "thiserror",
+ "tokio",
+ "tonic",
+ "tonic-build",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -2554,6 +2640,16 @@ dependencies = [
  "sha-1 0.8.2",
 ]
 
+[[package]]
+name = "petgraph"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
 [[package]]
 name = "phf"
 version = "0.8.0"
@@ -2681,6 +2777,57 @@ dependencies = [
  "unicode-xid 0.2.2",
 ]
 
+[[package]]
+name = "prost"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603"
+dependencies = [
+ "bytes",
+ "heck",
+ "itertools",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost",
+ "prost-types",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2 1.0.33",
+ "quote 1.0.10",
+ "syn 1.0.82",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
+dependencies = [
+ "bytes",
+ "prost",
+]
+
 [[package]]
 name = "quick-xml"
 version = "0.22.0"
@@ -2962,11 +3109,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89af431b8c46776b5071a9a739c2b5fadbed6be2c6158d1ac5f71c4da3d2261c"
 dependencies = [
  "async-trait",
+ "opentelemetry",
  "reqwest",
  "reqwest-middleware",
  "task-local-extensions",
  "tokio",
  "tracing",
+ "tracing-opentelemetry",
 ]
 
 [[package]]
@@ -3553,6 +3702,7 @@ dependencies = [
  "libc",
  "memchr",
  "mio 0.7.14",
+ "num_cpus",
  "once_cell",
  "parking_lot",
  "pin-project-lite",
@@ -3561,6 +3711,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-macros"
 version = "1.6.0"
@@ -3593,6 +3753,17 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "tokio-stream"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.6.9"
@@ -3616,6 +3787,76 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "796c5e1cd49905e65dd8e700d4cb1dffcbfdb4fc9d017de08c1a537afd83627c"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "base64 0.13.0",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "prost-derive",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tonic-build"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08"
+dependencies = [
+ "proc-macro2 1.0.33",
+ "prost-build",
+ "quote 1.0.10",
+ "syn 1.0.82",
+]
+
+[[package]]
+name = "tower"
+version = "0.4.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5651b5f6860a99bd1adb59dbfe1db8beb433e73709d9032b413a77e2fb7c066a"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "indexmap",
+ "pin-project",
+ "pin-project-lite",
+ "rand 0.8.4",
+ "slab",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"
+
 [[package]]
 name = "tower-service"
 version = "0.3.1"
@@ -3629,6 +3870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
 dependencies = [
  "cfg-if",
+ "log",
  "pin-project-lite",
  "tracing-attributes",
  "tracing-core",
@@ -3698,6 +3940,19 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-opentelemetry"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ffbf13a0f8b054a4e59df3a173b818e9c6177c02789871f2073977fd0062076"
+dependencies = [
+ "opentelemetry",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.3"
@@ -4027,6 +4282,17 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e"
 
+[[package]]
+name = "which"
+version = "4.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9"
+dependencies = [
+ "either",
+ "lazy_static",
+ "libc",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
diff --git a/Cargo.toml b/Cargo.toml
index 18bbf125..5af893af 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -62,12 +62,15 @@ tokio = { version = "1.14.0", features = ["sync"] }
 anyhow = "1.0.51"
 reqwest = { version = "0.11.7", features = ["json"] }
 reqwest-middleware = "0.1.3"
-reqwest-tracing = "0.2.0"
+reqwest-tracing = { version = "0.2.0", features = ["opentelemetry_0_16"] }
 activitystreams = "0.7.0-alpha.14"
 actix-rt = { version = "2.5.0", default-features = false }
 serde_json = { version = "1.0.72", features = ["preserve_order"] }
 clokwerk = "0.3.5"
 doku = "0.10.2"
+opentelemetry = { version = "0.16", features = ["rt-tokio"] }
+opentelemetry-otlp = "0.9"
+tracing-opentelemetry = "0.16"
 
 [dev-dependencies.cargo-husky]
 version = "1.5.0"
diff --git a/config/defaults.hjson b/config/defaults.hjson
index 9edf9fc0..663aa4b5 100644
--- a/config/defaults.hjson
+++ b/config/defaults.hjson
@@ -111,10 +111,11 @@
   # Whether the site is available over TLS. Needs to be true for federation to work.
   tls_enabled: true
   # Address where pictrs is available (for image hosting)
-  pictrs_url: "http:#localhost:8080"
+  pictrs_url: "http://localhost:8080"
   slur_filter: "(\bThis\b)|(\bis\b)|(\bsample\b)"
   # Maximum length of local community and user names
   actor_name_max_length: 20
   # Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object fetch through the search).
   http_fetch_retry_limit: 25
+  opentelemetry_url: "http://localhost:4317"
 }
diff --git a/crates/api_common/src/lib.rs b/crates/api_common/src/lib.rs
index cd854a6d..7ac95450 100644
--- a/crates/api_common/src/lib.rs
+++ b/crates/api_common/src/lib.rs
@@ -55,6 +55,7 @@ where
   res
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn is_mod_or_admin(
   pool: &DbPool,
   person_id: PersonId,
@@ -77,6 +78,7 @@ pub fn is_admin(local_user_view: &LocalUserView) -> Result<(), LemmyError> {
   Ok(())
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn get_post(post_id: PostId, pool: &DbPool) -> Result<Post, LemmyError> {
   blocking(pool, move |conn| Post::read(conn, post_id))
     .await?
@@ -84,6 +86,7 @@ pub async fn get_post(post_id: PostId, pool: &DbPool) -> Result<Post, LemmyError
     .map_err(|e| e.with_message("couldnt_find_post"))
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn mark_post_as_read(
   person_id: PersonId,
   post_id: PostId,
@@ -99,6 +102,7 @@ pub async fn mark_post_as_read(
   .map_err(|e| e.with_message("couldnt_mark_post_as_read"))
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn mark_post_as_unread(
   person_id: PersonId,
   post_id: PostId,
@@ -114,6 +118,7 @@ pub async fn mark_post_as_unread(
   .map_err(|e| e.with_message("couldnt_mark_post_as_read"))
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn get_local_user_view_from_jwt(
   jwt: &str,
   pool: &DbPool,
@@ -154,6 +159,7 @@ pub fn check_validator_time(
   }
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn get_local_user_view_from_jwt_opt(
   jwt: Option<&Sensitive<String>>,
   pool: &DbPool,
@@ -165,6 +171,7 @@ pub async fn get_local_user_view_from_jwt_opt(
   }
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn get_local_user_settings_view_from_jwt(
   jwt: &Sensitive<String>,
   pool: &DbPool,
@@ -189,6 +196,7 @@ pub async fn get_local_user_settings_view_from_jwt(
   Ok(local_user_view)
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn get_local_user_settings_view_from_jwt_opt(
   jwt: Option<&Sensitive<String>>,
   pool: &DbPool,
@@ -202,6 +210,7 @@ pub async fn get_local_user_settings_view_from_jwt_opt(
   }
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn check_community_ban(
   person_id: PersonId,
   community_id: CommunityId,
@@ -216,6 +225,7 @@ pub async fn check_community_ban(
   }
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn check_community_deleted_or_removed(
   community_id: CommunityId,
   pool: &DbPool,
@@ -239,6 +249,7 @@ pub fn check_post_deleted_or_removed(post: &Post) -> Result<(), LemmyError> {
   }
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn check_person_block(
   my_id: PersonId,
   potential_blocker_id: PersonId,
@@ -252,6 +263,7 @@ pub async fn check_person_block(
   }
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn check_downvotes_enabled(score: i16, pool: &DbPool) -> Result<(), LemmyError> {
   if score == -1 {
     let site = blocking(pool, Site::read_simple).await??;
@@ -262,6 +274,7 @@ pub async fn check_downvotes_enabled(score: i16, pool: &DbPool) -> Result<(), Le
   Ok(())
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn check_private_instance(
   local_user_view: &Option<LocalUserView>,
   pool: &DbPool,
@@ -275,6 +288,7 @@ pub async fn check_private_instance(
   Ok(())
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn build_federated_instances(
   pool: &DbPool,
   federation_config: &FederationConfig,
diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs
index bc2f29f7..276504be 100644
--- a/crates/api_crud/src/post/create.rs
+++ b/crates/api_crud/src/post/create.rs
@@ -31,7 +31,7 @@ use lemmy_utils::{
   LemmyError,
 };
 use lemmy_websocket::{send::send_post_ws_message, LemmyContext, UserOperationCrud};
-use tracing::warn;
+use tracing::{warn, Instrument};
 use url::Url;
 use webmention::{Webmention, WebmentionError};
 
@@ -132,7 +132,11 @@ impl PerformCrud for CreatePost {
       let mut webmention =
         Webmention::new::<Url>(updated_post.ap_id.clone().into(), url.clone().into())?;
       webmention.set_checked(true);
-      match webmention.send().await {
+      match webmention
+        .send()
+        .instrument(tracing::info_span!("Sending webmention"))
+        .await
+      {
         Ok(_) => {}
         Err(WebmentionError::NoEndpointDiscovered(_)) => {}
         Err(e) => warn!("Failed to send webmention: {}", e),
diff --git a/crates/apub_lib/src/activity_queue.rs b/crates/apub_lib/src/activity_queue.rs
index c02586ce..7357b2d6 100644
--- a/crates/apub_lib/src/activity_queue.rs
+++ b/crates/apub_lib/src/activity_queue.rs
@@ -80,12 +80,11 @@ async fn do_send(task: SendActivityTask, client: &ClientWithMiddleware) -> Resul
   match result {
     Ok(o) => {
       if !o.status().is_success() {
+        let status = o.status();
+        let text = o.text().await?;
         warn!(
           "Send {} to {} failed with status {}: {}",
-          task.activity_id,
-          task.inbox,
-          o.status(),
-          o.text().await?
+          task.activity_id, task.inbox, status, text
         );
       }
     }
diff --git a/crates/utils/src/request.rs b/crates/utils/src/request.rs
index 6af360a5..5ae87e98 100644
--- a/crates/utils/src/request.rs
+++ b/crates/utils/src/request.rs
@@ -18,6 +18,7 @@ struct SendError(pub String);
 #[error("Error receiving response, {0}")]
 pub struct RecvError(pub String);
 
+#[tracing::instrument(skip_all)]
 pub async fn retry<F, Fut, T>(f: F) -> Result<T, reqwest_middleware::Error>
 where
   F: Fn() -> Fut,
@@ -26,6 +27,7 @@ where
   retry_custom(|| async { Ok((f)().await) }).await
 }
 
+#[tracing::instrument(skip_all)]
 async fn retry_custom<F, Fut, T>(f: F) -> Result<T, reqwest_middleware::Error>
 where
   F: Fn() -> Fut,
@@ -61,6 +63,7 @@ pub struct SiteMetadata {
 }
 
 /// Fetches the post link html tags (like title, description, image, etc)
+#[tracing::instrument(skip_all)]
 pub async fn fetch_site_metadata(
   client: &ClientWithMiddleware,
   url: &Url,
@@ -159,6 +162,7 @@ pub(crate) struct PictrsFile {
   delete_token: String,
 }
 
+#[tracing::instrument(skip_all)]
 pub(crate) async fn fetch_pictrs(
   client: &ClientWithMiddleware,
   settings: &Settings,
@@ -192,6 +196,7 @@ pub(crate) async fn fetch_pictrs(
 
 /// Both are options, since the URL might be either an html page, or an image
 /// Returns the SiteMetadata, and a Pictrs URL, if there is a picture associated
+#[tracing::instrument(skip_all)]
 pub async fn fetch_site_data(
   client: &ClientWithMiddleware,
   settings: &Settings,
@@ -242,6 +247,7 @@ pub async fn fetch_site_data(
   }
 }
 
+#[tracing::instrument(skip_all)]
 async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Result<(), LemmyError> {
   let response = client.get(url.as_str()).send().await?;
   if response
diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs
index 2de87c5b..1f712134 100644
--- a/crates/utils/src/settings/structs.rs
+++ b/crates/utils/src/settings/structs.rs
@@ -49,6 +49,10 @@ pub struct Settings {
   /// Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object fetch through the search).
   #[default(25)]
   pub http_fetch_retry_limit: i32,
+
+  #[default(None)]
+  #[doku(example = "http://localhost:4317")]
+  pub opentelemetry_url: Option<String>,
 }
 
 #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)]
diff --git a/crates/websocket/Cargo.toml b/crates/websocket/Cargo.toml
index c59eb458..541aafde 100644
--- a/crates/websocket/Cargo.toml
+++ b/crates/websocket/Cargo.toml
@@ -34,3 +34,5 @@ strum_macros = "0.23.1"
 chrono = { version = "0.4.19", features = ["serde"] }
 actix-web = { version = "4.0.0-beta.18", default-features = false, features = ["rustls"] }
 actix-web-actors = { version = "4.0.0-beta.8", default-features = false }
+opentelemetry = "0.16"
+tracing-opentelemetry = "0.16"
diff --git a/crates/websocket/src/handlers.rs b/crates/websocket/src/handlers.rs
index f35e8b31..94ca8bf9 100644
--- a/crates/websocket/src/handlers.rs
+++ b/crates/websocket/src/handlers.rs
@@ -6,9 +6,11 @@ use crate::{
 use actix::{Actor, Context, Handler, ResponseFuture};
 use lemmy_db_schema::naive_now;
 use lemmy_utils::ConnectionId;
+use opentelemetry::trace::TraceContextExt;
 use rand::Rng;
 use serde::Serialize;
 use tracing::{error, info};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 /// Make actor from `ChatServer`
 impl Actor for ChatServer {
@@ -62,27 +64,48 @@ impl Handler<Disconnect> for ChatServer {
   }
 }
 
+fn root_span() -> tracing::Span {
+  let span = tracing::info_span!(
+    parent: None,
+    "Websocket Request",
+    trace_id = tracing::field::Empty,
+  );
+  {
+    let trace_id = span.context().span().span_context().trace_id().to_hex();
+    span.record("trace_id", &tracing::field::display(trace_id));
+  }
+
+  span
+}
+
 /// Handler for Message message.
 impl Handler<StandardMessage> for ChatServer {
   type Result = ResponseFuture<Result<String, std::convert::Infallible>>;
 
   fn handle(&mut self, msg: StandardMessage, ctx: &mut Context<Self>) -> Self::Result {
     let fut = self.parse_json_message(msg, ctx);
-    Box::pin(async move {
-      match fut.await {
-        Ok(m) => {
-          // info!("Message Sent: {}", m);
-          Ok(m)
-        }
-        Err(e) => {
-          error!("Error during message handling {}", e);
-          Ok(
-            e.to_json()
-              .unwrap_or_else(|_| String::from(r#"{"error":"failed to serialize json"}"#)),
-          )
+    let span = root_span();
+
+    use tracing::Instrument;
+
+    Box::pin(
+      async move {
+        match fut.await {
+          Ok(m) => {
+            // info!("Message Sent: {}", m);
+            Ok(m)
+          }
+          Err(e) => {
+            error!("Error during message handling {}", e);
+            Ok(
+              e.to_json()
+                .unwrap_or_else(|_| String::from(r#"{"error":"failed to serialize json"}"#)),
+            )
+          }
         }
       }
-    })
+      .instrument(span),
+    )
   }
 }
 
diff --git a/crates/websocket/src/send.rs b/crates/websocket/src/send.rs
index e7f265b5..36e93fb6 100644
--- a/crates/websocket/src/send.rs
+++ b/crates/websocket/src/send.rs
@@ -31,6 +31,7 @@ use lemmy_db_views::{
 use lemmy_db_views_actor::community_view::CommunityView;
 use lemmy_utils::{utils::MentionData, ConnectionId, LemmyError};
 
+#[tracing::instrument(skip_all)]
 pub async fn send_post_ws_message<OP: ToString + Send + OperationType + 'static>(
   post_id: PostId,
   op: OP,
@@ -56,6 +57,7 @@ pub async fn send_post_ws_message<OP: ToString + Send + OperationType + 'static>
 
 // TODO: in many call sites in apub crate, we are setting an empty vec for recipient_ids,
 //       we should get the actual recipient actors from somewhere
+#[tracing::instrument(skip_all)]
 pub async fn send_comment_ws_message_simple<OP: ToString + Send + OperationType + 'static>(
   comment_id: CommentId,
   op: OP,
@@ -64,6 +66,7 @@ pub async fn send_comment_ws_message_simple<OP: ToString + Send + OperationType
   send_comment_ws_message(comment_id, op, None, None, None, vec![], context).await
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn send_comment_ws_message<OP: ToString + Send + OperationType + 'static>(
   comment_id: CommentId,
   op: OP,
@@ -102,6 +105,7 @@ pub async fn send_comment_ws_message<OP: ToString + Send + OperationType + 'stat
   Ok(res)
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn send_community_ws_message<OP: ToString + Send + OperationType + 'static>(
   community_id: CommunityId,
   op: OP,
@@ -130,6 +134,7 @@ pub async fn send_community_ws_message<OP: ToString + Send + OperationType + 'st
   Ok(res)
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn send_pm_ws_message<OP: ToString + Send + OperationType + 'static>(
   private_message_id: PrivateMessageId,
   op: OP,
@@ -168,6 +173,7 @@ pub async fn send_pm_ws_message<OP: ToString + Send + OperationType + 'static>(
   Ok(res)
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn send_local_notifs(
   mentions: Vec<MentionData>,
   comment: &Comment,
diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml
index ca88ac38..9bfcbc44 100644
--- a/docker/dev/docker-compose.yml
+++ b/docker/dev/docker-compose.yml
@@ -24,6 +24,7 @@ services:
     depends_on: 
       - pictrs
       - postgres
+      - otel
 
   lemmy-ui:
     image: dessalines/lemmy-ui:0.14.3
@@ -51,6 +52,31 @@ services:
   pictrs:
     image: asonix/pictrs:0.3.0-beta.11
     user: 991:991
+    environment:
+      - PICTRS_OPENTELEMETRY_URL=http://otel:4137
     volumes:
       - ./volumes/pictrs:/mnt
     restart: always
+    depends_on:
+      - otel
+
+  otel:
+    image: otel/opentelemetry-collector:latest
+    command: --config otel-local-config.yaml
+    ports:
+      - "4317:4317"
+    volumes:
+      - type: bind
+        source: ./otel.yml
+        target: /otel-local-config.yaml
+    restart: always
+    depends_on:
+      - jaeger
+
+  jaeger:
+    image: jaegertracing/all-in-one:1
+    ports:
+      - "14250:14250"
+      # To view traces, visit http://localhost:16686
+      - "16686:16686"
+    restart: always
diff --git a/docker/dev/otel.yml b/docker/dev/otel.yml
new file mode 100644
index 00000000..8270b089
--- /dev/null
+++ b/docker/dev/otel.yml
@@ -0,0 +1,25 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4137
+
+processors:
+  batch:
+
+exporters:
+  logging:
+  jaeger:
+    endpoint: jaeger:14250
+    insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers:
+        - otlp
+      processors:
+        - batch
+      exporters:
+        - logging
+        - jaeger
diff --git a/docker/lemmy.hjson b/docker/lemmy.hjson
index 54975fec..779c8d2c 100644
--- a/docker/lemmy.hjson
+++ b/docker/lemmy.hjson
@@ -11,6 +11,8 @@
     site_name: "lemmy-test"
   }
 
+  opentelemetry_url: "http://otel:4137"
+
   # the domain name of your instance (eg "lemmy.ml")
   hostname: "my_domain"
   # address where lemmy should listen for incoming requests
diff --git a/scripts/update_config_defaults.sh b/scripts/update_config_defaults.sh
index 0aca7160..f15f54a3 100755
--- a/scripts/update_config_defaults.sh
+++ b/scripts/update_config_defaults.sh
@@ -5,7 +5,7 @@ dest=${1-config/defaults.hjson}
 
 cargo run -- --print-config-docs > "$dest"
 # replace // comments with #
-sed -i "s/\/\//#/" "$dest"
+sed -i "s/^\([[:space:]]*\)\/\//\1#/" "$dest"
 # remove trailing commas
 sed -i "s/,\$//" "$dest"
 # remove quotes around json keys
diff --git a/src/lib.rs b/src/lib.rs
index 62d07166..29b14469 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,14 +5,21 @@ pub mod root_span_builder;
 pub mod scheduled_tasks;
 
 use lemmy_utils::LemmyError;
+use opentelemetry::{
+  sdk::{propagation::TraceContextPropagator, Resource},
+  KeyValue,
+};
+use opentelemetry_otlp::WithExportConfig;
 use tracing::subscriber::set_global_default;
 use tracing_error::ErrorLayer;
 use tracing_log::LogTracer;
 use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
 
-pub fn init_tracing() -> Result<(), LemmyError> {
+pub fn init_tracing(opentelemetry_url: Option<&str>) -> Result<(), LemmyError> {
   LogTracer::init()?;
 
+  opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
+
   let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
   let format_layer = tracing_subscriber::fmt::layer();
 
@@ -21,7 +28,28 @@ pub fn init_tracing() -> Result<(), LemmyError> {
     .with(format_layer)
     .with(ErrorLayer::default());
 
-  set_global_default(subscriber)?;
+  if let Some(url) = opentelemetry_url {
+    let tracer = opentelemetry_otlp::new_pipeline()
+      .tracing()
+      .with_trace_config(
+        opentelemetry::sdk::trace::config()
+          .with_resource(Resource::new(vec![KeyValue::new("service.name", "lemmy")])),
+      )
+      .with_exporter(
+        opentelemetry_otlp::new_exporter()
+          .tonic()
+          .with_endpoint(url),
+      )
+      .install_batch(opentelemetry::runtime::Tokio)?;
+
+    let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
+
+    let subscriber = subscriber.with(otel_layer);
+
+    set_global_default(subscriber)?;
+  } else {
+    set_global_default(subscriber)?;
+  }
 
   Ok(())
 }
diff --git a/src/main.rs b/src/main.rs
index 252d37a3..2d3adc58 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -49,10 +49,10 @@ async fn main() -> Result<(), LemmyError> {
     return Ok(());
   }
 
-  init_tracing()?;
-
   let settings = Settings::init().expect("Couldn't initialize settings.");
 
+  init_tracing(settings.opentelemetry_url.as_deref())?;
+
   // Set up the r2d2 connection pool
   let db_url = match get_database_url_from_env() {
     Ok(url) => url,