From: nutomic Date: Thu, 4 Feb 2021 16:34:58 +0000 (+0000) Subject: Store activitypub endpoints in database (#162) X-Git-Url: http://these/git/%7B%60/feeds/%7BdonateLemmyUrl%7D?a=commitdiff_plain;h=1a4e35eb509c2b4836092526ffa11d2e1a04b1cd;p=lemmy.git Store activitypub endpoints in database (#162) Address review comments Store Activitypub urls in database (fixes #808) Co-authored-by: Felix Ableitner Reviewed-on: https://yerbamate.ml/LemmyNet/lemmy/pulls/162 Co-Authored-By: nutomic Co-Committed-By: nutomic --- diff --git a/Cargo.lock b/Cargo.lock index c6b9a0d9..8553c1f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1887,6 +1887,7 @@ dependencies = [ "lemmy_db_schema", "log", "serde 1.0.123", + "url", ] [[package]] diff --git a/crates/api/src/comment.rs b/crates/api/src/comment.rs index 56c0ce62..5d798d1d 100644 --- a/crates/api/src/comment.rs +++ b/crates/api/src/comment.rs @@ -9,7 +9,7 @@ use crate::{ Perform, }; use actix_web::web::Data; -use lemmy_apub::{ApubLikeableType, ApubObjectType}; +use lemmy_apub::{generate_apub_endpoint, ApubLikeableType, ApubObjectType, EndpointType}; use lemmy_db_queries::{ source::comment::Comment_, Crud, @@ -26,7 +26,6 @@ use lemmy_db_views::{ }; use lemmy_structs::{blocking, comment::*, send_local_notifs}; use lemmy_utils::{ - apub::{make_apub_endpoint, EndpointType}, utils::{remove_slurs, scrape_text_for_mentions}, APIError, ConnectionId, @@ -104,16 +103,17 @@ impl Perform for CreateComment { // Necessary to update the ap_id let inserted_comment_id = inserted_comment.id; - let updated_comment: Comment = match blocking(context.pool(), move |conn| { - let apub_id = - make_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string()).to_string(); - Comment::update_ap_id(&conn, inserted_comment_id, apub_id) - }) - .await? - { - Ok(comment) => comment, - Err(_e) => return Err(APIError::err("couldnt_create_comment").into()), - }; + let updated_comment: Comment = + match blocking(context.pool(), move |conn| -> Result { + let apub_id = + generate_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string())?; + Ok(Comment::update_ap_id(&conn, inserted_comment_id, apub_id)?) + }) + .await? + { + Ok(comment) => comment, + Err(_e) => return Err(APIError::err("couldnt_create_comment").into()), + }; updated_comment.send_create(&user, context).await?; diff --git a/crates/api/src/community.rs b/crates/api/src/community.rs index d2e0e841..1c47ef8e 100644 --- a/crates/api/src/community.rs +++ b/crates/api/src/community.rs @@ -9,7 +9,14 @@ use crate::{ }; use actix_web::web::Data; use anyhow::Context; -use lemmy_apub::ActorType; +use lemmy_apub::{ + generate_apub_endpoint, + generate_followers_url, + generate_inbox_url, + generate_shared_inbox_url, + ActorType, + EndpointType, +}; use lemmy_db_queries::{ diesel_option_overwrite, source::{ @@ -38,7 +45,7 @@ use lemmy_db_views_actor::{ }; use lemmy_structs::{blocking, community::*}; use lemmy_utils::{ - apub::{generate_actor_keypair, make_apub_endpoint, EndpointType}, + apub::generate_actor_keypair, location_info, utils::{check_slurs, check_slurs_opt, is_valid_community_name, naive_from_unix}, APIError, @@ -137,10 +144,10 @@ impl Perform for CreateCommunity { } // Double check for duplicate community actor_ids - let actor_id = make_apub_endpoint(EndpointType::Community, &data.name); - let actor_id_cloned = actor_id.to_owned(); + let community_actor_id = generate_apub_endpoint(EndpointType::Community, &data.name)?; + let actor_id_cloned = community_actor_id.to_owned(); let community_dupe = blocking(context.pool(), move |conn| { - Community::read_from_apub_id(conn, &actor_id_cloned.into()) + Community::read_from_apub_id(conn, &actor_id_cloned) }) .await?; if community_dupe.is_ok() { @@ -169,12 +176,15 @@ impl Perform for CreateCommunity { deleted: None, nsfw: data.nsfw, updated: None, - actor_id: Some(actor_id.into()), + actor_id: Some(community_actor_id.to_owned()), local: true, private_key: Some(keypair.private_key), public_key: Some(keypair.public_key), last_refreshed_at: None, published: None, + followers_url: Some(generate_followers_url(&community_actor_id)?), + inbox_url: Some(generate_inbox_url(&community_actor_id)?), + shared_inbox_url: Some(Some(generate_shared_inbox_url(&community_actor_id)?)), }; let inserted_community = match blocking(context.pool(), move |conn| { @@ -275,6 +285,9 @@ impl Perform for EditCommunity { public_key: read_community.public_key, last_refreshed_at: None, published: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let community_id = data.community_id; diff --git a/crates/api/src/post.rs b/crates/api/src/post.rs index 4a2f14ce..5ab461dc 100644 --- a/crates/api/src/post.rs +++ b/crates/api/src/post.rs @@ -9,7 +9,7 @@ use crate::{ Perform, }; use actix_web::web::Data; -use lemmy_apub::{ApubLikeableType, ApubObjectType}; +use lemmy_apub::{generate_apub_endpoint, ApubLikeableType, ApubObjectType, EndpointType}; use lemmy_db_queries::{ source::post::Post_, Crud, @@ -38,7 +38,6 @@ use lemmy_db_views_actor::{ }; use lemmy_structs::{blocking, post::*}; use lemmy_utils::{ - apub::{make_apub_endpoint, EndpointType}, request::fetch_iframely_and_pictrs_data, utils::{check_slurs, check_slurs_opt, is_valid_post_title}, APIError, @@ -115,10 +114,9 @@ impl Perform for CreatePost { }; let inserted_post_id = inserted_post.id; - let updated_post = match blocking(context.pool(), move |conn| { - let apub_id = - make_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string()).to_string(); - Post::update_ap_id(conn, inserted_post_id, apub_id) + let updated_post = match blocking(context.pool(), move |conn| -> Result { + let apub_id = generate_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string())?; + Ok(Post::update_ap_id(conn, inserted_post_id, apub_id)?) }) .await? { diff --git a/crates/api/src/user.rs b/crates/api/src/user.rs index 4ec93054..fe124f23 100644 --- a/crates/api/src/user.rs +++ b/crates/api/src/user.rs @@ -13,7 +13,14 @@ use anyhow::Context; use bcrypt::verify; use captcha::{gen, Difficulty}; use chrono::Duration; -use lemmy_apub::ApubObjectType; +use lemmy_apub::{ + generate_apub_endpoint, + generate_followers_url, + generate_inbox_url, + generate_shared_inbox_url, + ApubObjectType, + EndpointType, +}; use lemmy_db_queries::{ diesel_option_overwrite, source::{ @@ -61,7 +68,7 @@ use lemmy_db_views_actor::{ }; use lemmy_structs::{blocking, send_email_to_user, user::*}; use lemmy_utils::{ - apub::{generate_actor_keypair, make_apub_endpoint, EndpointType}, + apub::generate_actor_keypair, email::send_email, location_info, settings::Settings, @@ -179,6 +186,7 @@ impl Perform for Register { if !is_valid_username(&data.username) { return Err(APIError::err("invalid_username").into()); } + let user_actor_id = generate_apub_endpoint(EndpointType::User, &data.username)?; // Register the new user let user_form = UserForm { @@ -200,12 +208,14 @@ impl Perform for Register { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: Some(make_apub_endpoint(EndpointType::User, &data.username).into()), + actor_id: Some(user_actor_id.clone()), bio: None, local: true, private_key: Some(user_keypair.private_key), public_key: Some(user_keypair.public_key), last_refreshed_at: None, + inbox_url: Some(generate_inbox_url(&user_actor_id)?), + shared_inbox_url: Some(Some(generate_shared_inbox_url(&user_actor_id)?)), }; // Create the user @@ -236,6 +246,7 @@ impl Perform for Register { Ok(c) => c, Err(_e) => { let default_community_name = "main"; + let actor_id = generate_apub_endpoint(EndpointType::Community, default_community_name)?; let community_form = CommunityForm { name: default_community_name.to_string(), title: "The Default Community".to_string(), @@ -246,9 +257,7 @@ impl Perform for Register { removed: None, deleted: None, updated: None, - actor_id: Some( - make_apub_endpoint(EndpointType::Community, default_community_name).into(), - ), + actor_id: Some(actor_id.to_owned()), local: true, private_key: Some(main_community_keypair.private_key), public_key: Some(main_community_keypair.public_key), @@ -256,6 +265,9 @@ impl Perform for Register { published: None, icon: None, banner: None, + followers_url: Some(generate_followers_url(&actor_id)?), + inbox_url: Some(generate_inbox_url(&actor_id)?), + shared_inbox_url: Some(Some(generate_shared_inbox_url(&actor_id)?)), }; blocking(context.pool(), move |conn| { Community::create(conn, &community_form) @@ -420,6 +432,7 @@ impl Perform for SaveUserSettings { matrix_user_id, avatar, banner, + inbox_url: None, password_encrypted, preferred_username, published: Some(user.published), @@ -439,6 +452,7 @@ impl Perform for SaveUserSettings { private_key: user.private_key, public_key: user.public_key, last_refreshed_at: None, + shared_inbox_url: None, }; let res = blocking(context.pool(), move |conn| { @@ -1036,14 +1050,20 @@ impl Perform for CreatePrivateMessage { }; let inserted_private_message_id = inserted_private_message.id; - let updated_private_message = match blocking(context.pool(), move |conn| { - let apub_id = make_apub_endpoint( - EndpointType::PrivateMessage, - &inserted_private_message_id.to_string(), - ) - .to_string(); - PrivateMessage::update_ap_id(&conn, inserted_private_message_id, apub_id) - }) + let updated_private_message = match blocking( + context.pool(), + move |conn| -> Result { + let apub_id = generate_apub_endpoint( + EndpointType::PrivateMessage, + &inserted_private_message_id.to_string(), + )?; + Ok(PrivateMessage::update_ap_id( + &conn, + inserted_private_message_id, + apub_id, + )?) + }, + ) .await? { Ok(private_message) => private_message, diff --git a/crates/apub/src/activities/send/comment.rs b/crates/apub/src/activities/send/comment.rs index 323b851f..f007cda4 100644 --- a/crates/apub/src/activities/send/comment.rs +++ b/crates/apub/src/activities/send/comment.rs @@ -351,7 +351,7 @@ async fn collect_non_local_mentions( let parent_creator = get_comment_parent_creator(context.pool(), comment).await?; let mut addressed_ccs = vec![community.actor_id(), parent_creator.actor_id()]; // Note: dont include community inbox here, as we send to it separately with `send_to_community()` - let mut inboxes = vec![parent_creator.get_shared_inbox_url()?]; + let mut inboxes = vec![parent_creator.get_shared_inbox_or_inbox_url()]; // Add the mention tag let mut tags = Vec::new(); @@ -370,7 +370,7 @@ async fn collect_non_local_mentions( addressed_ccs.push(actor_id.to_owned().to_string().parse()?); let mention_user = get_or_fetch_and_upsert_user(&actor_id, context, &mut 0).await?; - inboxes.push(mention_user.get_shared_inbox_url()?); + inboxes.push(mention_user.get_shared_inbox_or_inbox_url()); let mut mention_tag = Mention::new(); mention_tag.set_href(actor_id).set_name(mention.full_name()); diff --git a/crates/apub/src/activities/send/community.rs b/crates/apub/src/activities/send/community.rs index d79cfe06..a574c7b8 100644 --- a/crates/apub/src/activities/send/community.rs +++ b/crates/apub/src/activities/send/community.rs @@ -27,16 +27,18 @@ use lemmy_db_queries::DbPool; use lemmy_db_schema::source::community::Community; use lemmy_db_views_actor::community_follower_view::CommunityFollowerView; use lemmy_structs::blocking; -use lemmy_utils::{location_info, settings::Settings, LemmyError}; +use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; use url::Url; #[async_trait::async_trait(?Send)] impl ActorType for Community { + fn is_local(&self) -> bool { + self.local + } fn actor_id(&self) -> Url { self.actor_id.to_owned().into_inner() } - fn public_key(&self) -> Option { self.public_key.to_owned() } @@ -44,6 +46,14 @@ impl ActorType for Community { self.private_key.to_owned() } + fn get_shared_inbox_or_inbox_url(&self) -> Url { + self + .shared_inbox_url + .clone() + .unwrap_or_else(|| self.inbox_url.to_owned()) + .into() + } + async fn send_follow( &self, _follow_actor_id: &Url, @@ -81,7 +91,7 @@ impl ActorType for Community { .set_id(generate_activity_id(AcceptType::Accept)?) .set_to(user.actor_id()); - send_activity_single_dest(accept, self, user.get_inbox_url()?, context).await?; + send_activity_single_dest(accept, self, user.inbox_url.into(), context).await?; Ok(()) } @@ -92,7 +102,7 @@ impl ActorType for Community { .set_many_contexts(lemmy_context()?) .set_id(generate_activity_id(DeleteType::Delete)?) .set_to(public()) - .set_many_ccs(vec![self.get_followers_url()?]); + .set_many_ccs(vec![self.followers_url.clone().into_inner()]); send_to_community_followers(delete, self, context).await?; Ok(()) @@ -105,14 +115,14 @@ impl ActorType for Community { .set_many_contexts(lemmy_context()?) .set_id(generate_activity_id(DeleteType::Delete)?) .set_to(public()) - .set_many_ccs(vec![self.get_followers_url()?]); + .set_many_ccs(vec![self.followers_url.clone().into_inner()]); let mut undo = Undo::new(self.actor_id(), delete.into_any_base()?); undo .set_many_contexts(lemmy_context()?) .set_id(generate_activity_id(UndoType::Undo)?) .set_to(public()) - .set_many_ccs(vec![self.get_followers_url()?]); + .set_many_ccs(vec![self.followers_url.clone().into_inner()]); send_to_community_followers(undo, self, context).await?; Ok(()) @@ -125,7 +135,7 @@ impl ActorType for Community { .set_many_contexts(lemmy_context()?) .set_id(generate_activity_id(RemoveType::Remove)?) .set_to(public()) - .set_many_ccs(vec![self.get_followers_url()?]); + .set_many_ccs(vec![self.followers_url.clone().into_inner()]); send_to_community_followers(remove, self, context).await?; Ok(()) @@ -138,7 +148,7 @@ impl ActorType for Community { .set_many_contexts(lemmy_context()?) .set_id(generate_activity_id(RemoveType::Remove)?) .set_to(public()) - .set_many_ccs(vec![self.get_followers_url()?]); + .set_many_ccs(vec![self.followers_url.clone().into_inner()]); // Undo that fake activity let mut undo = Undo::new(self.actor_id(), remove.into_any_base()?); @@ -146,7 +156,7 @@ impl ActorType for Community { .set_many_contexts(lemmy_context()?) .set_id(generate_activity_id(LikeType::Like)?) .set_to(public()) - .set_many_ccs(vec![self.get_followers_url()?]); + .set_many_ccs(vec![self.followers_url.clone().into_inner()]); send_to_community_followers(undo, self, context).await?; Ok(()) @@ -164,7 +174,7 @@ impl ActorType for Community { .set_many_contexts(lemmy_context()?) .set_id(generate_activity_id(AnnounceType::Announce)?) .set_to(public()) - .set_many_ccs(vec![self.get_followers_url()?]); + .set_many_ccs(vec![self.followers_url.clone().into_inner()]); send_to_community_followers(announce, self, context).await?; @@ -172,38 +182,21 @@ impl ActorType for Community { } /// For a given community, returns the inboxes of all followers. - /// - /// TODO: this function is very badly implemented, we should just store shared_inbox_url in - /// CommunityFollowerView async fn get_follower_inboxes(&self, pool: &DbPool) -> Result, LemmyError> { let id = self.id; - let inboxes = blocking(pool, move |conn| { + let follows = blocking(pool, move |conn| { CommunityFollowerView::for_community(conn, id) }) .await??; - let inboxes = inboxes + let inboxes = follows .into_iter() - .filter(|i| !i.follower.local) - .map(|u| -> Result { - let url = u.follower.actor_id.into_inner(); - let domain = url.domain().context(location_info!())?; - let port = if let Some(port) = url.port() { - format!(":{}", port) - } else { - "".to_string() - }; - Ok(Url::parse(&format!( - "{}://{}{}/inbox", - Settings::get().get_protocol_string(), - domain, - port, - ))?) - }) - .filter_map(Result::ok) + .filter(|f| !f.follower.local) + .map(|f| f.follower.shared_inbox_url.unwrap_or(f.follower.inbox_url)) + .map(|i| i.into_inner()) + .unique() // Don't send to blocked instances .filter(|inbox| check_is_apub_id_valid(inbox).is_ok()) - .unique() .collect(); Ok(inboxes) diff --git a/crates/apub/src/activities/send/private_message.rs b/crates/apub/src/activities/send/private_message.rs index 6bff815a..31184a70 100644 --- a/crates/apub/src/activities/send/private_message.rs +++ b/crates/apub/src/activities/send/private_message.rs @@ -41,7 +41,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(CreateType::Create)?) .set_to(recipient.actor_id()); - send_activity_single_dest(create, creator, recipient.get_inbox_url()?, context).await?; + send_activity_single_dest(create, creator, recipient.inbox_url.into(), context).await?; Ok(()) } @@ -61,7 +61,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(UpdateType::Update)?) .set_to(recipient.actor_id()); - send_activity_single_dest(update, creator, recipient.get_inbox_url()?, context).await?; + send_activity_single_dest(update, creator, recipient.inbox_url.into(), context).await?; Ok(()) } @@ -78,7 +78,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(DeleteType::Delete)?) .set_to(recipient.actor_id()); - send_activity_single_dest(delete, creator, recipient.get_inbox_url()?, context).await?; + send_activity_single_dest(delete, creator, recipient.inbox_url.into(), context).await?; Ok(()) } @@ -109,7 +109,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(UndoType::Undo)?) .set_to(recipient.actor_id()); - send_activity_single_dest(undo, creator, recipient.get_inbox_url()?, context).await?; + send_activity_single_dest(undo, creator, recipient.inbox_url.into(), context).await?; Ok(()) } diff --git a/crates/apub/src/activities/send/user.rs b/crates/apub/src/activities/send/user.rs index 3eac6f2d..1847ec5c 100644 --- a/crates/apub/src/activities/send/user.rs +++ b/crates/apub/src/activities/send/user.rs @@ -25,6 +25,9 @@ use url::Url; #[async_trait::async_trait(?Send)] impl ActorType for User_ { + fn is_local(&self) -> bool { + self.local + } fn actor_id(&self) -> Url { self.actor_id.to_owned().into_inner() } @@ -37,6 +40,14 @@ impl ActorType for User_ { self.private_key.to_owned() } + fn get_shared_inbox_or_inbox_url(&self) -> Url { + self + .shared_inbox_url + .clone() + .unwrap_or_else(|| self.inbox_url.to_owned()) + .into() + } + /// As a given local user, send out a follow request to a remote community. async fn send_follow( &self, @@ -65,7 +76,7 @@ impl ActorType for User_ { .set_id(generate_activity_id(FollowType::Follow)?) .set_to(community.actor_id()); - send_activity_single_dest(follow, self, community.get_inbox_url()?, context).await?; + send_activity_single_dest(follow, self, community.inbox_url.into(), context).await?; Ok(()) } @@ -96,7 +107,7 @@ impl ActorType for User_ { .set_id(generate_activity_id(UndoType::Undo)?) .set_to(community.actor_id()); - send_activity_single_dest(undo, self, community.get_inbox_url()?, context).await?; + send_activity_single_dest(undo, self, community.inbox_url.into(), context).await?; Ok(()) } diff --git a/crates/apub/src/activity_queue.rs b/crates/apub/src/activity_queue.rs index 1e1a6a5a..c0c4ac46 100644 --- a/crates/apub/src/activity_queue.rs +++ b/crates/apub/src/activity_queue.rs @@ -94,7 +94,7 @@ where .collect(); debug!( "Sending activity {:?} to followers of {}", - &activity.id_unchecked(), + &activity.id_unchecked().map(|i| i.to_string()), &community.actor_id ); @@ -135,7 +135,7 @@ where .send_announce(activity.into_any_base()?, context) .await?; } else { - let inbox = community.get_shared_inbox_url()?; + let inbox = community.get_shared_inbox_or_inbox_url(); check_is_apub_id_valid(&inbox)?; debug!( "Sending activity {:?} to community {}", diff --git a/crates/apub/src/fetcher/community.rs b/crates/apub/src/fetcher/community.rs index 7547e0db..cb9ec865 100644 --- a/crates/apub/src/fetcher/community.rs +++ b/crates/apub/src/fetcher/community.rs @@ -7,10 +7,10 @@ use crate::{ }, inbox::user_inbox::receive_announce, objects::FromApub, - ActorType, GroupExt, }; use activitystreams::{ + actor::ApActorExt, collection::{CollectionExt, OrderedCollection}, object::ObjectExt, }; @@ -116,7 +116,8 @@ async fn fetch_remote_community( // only fetch outbox for new communities, otherwise this can create an infinite loop if old_community.is_none() { - fetch_community_outbox(context, &community, recursion_counter).await? + let outbox = group.inner.outbox()?.context(location_info!())?; + fetch_community_outbox(context, outbox, &community, recursion_counter).await? } Ok(community) @@ -124,15 +125,12 @@ async fn fetch_remote_community( async fn fetch_community_outbox( context: &LemmyContext, + outbox: &Url, community: &Community, recursion_counter: &mut i32, ) -> Result<(), LemmyError> { - let outbox = fetch_remote_object::( - context.client(), - &community.get_outbox_url()?, - recursion_counter, - ) - .await?; + let outbox = + fetch_remote_object::(context.client(), outbox, recursion_counter).await?; let outbox_activities = outbox.items().context(location_info!())?.clone(); let mut outbox_activities = outbox_activities.many().context(location_info!())?; if outbox_activities.len() > 20 { diff --git a/crates/apub/src/http/community.rs b/crates/apub/src/http/community.rs index 8d6549ad..964ac2a1 100644 --- a/crates/apub/src/http/community.rs +++ b/crates/apub/src/http/community.rs @@ -60,7 +60,7 @@ pub async fn get_apub_community_followers( let mut collection = UnorderedCollection::new(); collection .set_many_contexts(lemmy_context()?) - .set_id(community.get_followers_url()?) + .set_id(community.followers_url.into()) .set_total_items(community_followers.len() as u64); Ok(create_apub_response(&collection)) } diff --git a/crates/apub/src/inbox/mod.rs b/crates/apub/src/inbox/mod.rs index 1e3574bc..765d5dff 100644 --- a/crates/apub/src/inbox/mod.rs +++ b/crates/apub/src/inbox/mod.rs @@ -12,7 +12,11 @@ use activitystreams::{ }; use actix_web::HttpRequest; use anyhow::{anyhow, Context}; -use lemmy_db_queries::{source::activity::Activity_, ApubObject, DbPool}; +use lemmy_db_queries::{ + source::{activity::Activity_, community::Community_}, + ApubObject, + DbPool, +}; use lemmy_db_schema::source::{activity::Activity, community::Community, user::User_}; use lemmy_structs::blocking; use lemmy_utils::{location_info, settings::Settings, LemmyError}; @@ -141,16 +145,15 @@ pub(crate) async fn is_addressed_to_community_followers( pool: &DbPool, ) -> Result, LemmyError> { for url in to_and_cc { - let url = url.to_string(); - // TODO: extremely hacky, we should just store the followers url for each community in the db - if url.ends_with("/followers") { - let community_url = Url::parse(&url.replace("/followers", ""))?; - let community = blocking(&pool, move |conn| { - Community::read_from_apub_id(&conn, &community_url.into()) - }) - .await??; - if !community.local { - return Ok(Some(community)); + let url = url.to_owned().into(); + let community = blocking(&pool, move |conn| { + // ignore errors here, because the current url might not actually be a followers url + Community::read_from_followers_url(&conn, &url).ok() + }) + .await?; + if let Some(c) = community { + if !c.local { + return Ok(Some(c)); } } } diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index f76a0b8f..9a9507eb 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -140,6 +140,7 @@ pub trait ApubLikeableType { /// implemented by all actors. #[async_trait::async_trait(?Send)] pub trait ActorType { + fn is_local(&self) -> bool; fn actor_id(&self) -> Url; // TODO: every actor should have a public key, so this shouldnt be an option (needs to be fixed in db) @@ -178,32 +179,15 @@ pub trait ActorType { /// For a given community, returns the inboxes of all followers. async fn get_follower_inboxes(&self, pool: &DbPool) -> Result, LemmyError>; - // TODO move these to the db rows - fn get_inbox_url(&self) -> Result { - Url::parse(&format!("{}/inbox", &self.actor_id())) - } - - fn get_shared_inbox_url(&self) -> Result { - let actor_id = self.actor_id(); - let url = format!( - "{}://{}{}/inbox", - &actor_id.scheme(), - &actor_id.host_str().context(location_info!())?, - if let Some(port) = actor_id.port() { - format!(":{}", port) - } else { - "".to_string() - }, - ); - Ok(Url::parse(&url)?) - } - - fn get_outbox_url(&self) -> Result { - Url::parse(&format!("{}/outbox", &self.actor_id())) - } + fn get_shared_inbox_or_inbox_url(&self) -> Url; - fn get_followers_url(&self) -> Result { - Url::parse(&format!("{}/followers", &self.actor_id())) + /// Outbox URL is not generally used by Lemmy, so it can be generated on the fly (but only for + /// local actors). + fn get_outbox_url(&self) -> Result { + if !self.is_local() { + return Err(anyhow!("get_outbox_url() called for remote actor").into()); + } + Ok(Url::parse(&format!("{}/outbox", &self.actor_id()))?) } fn get_public_key_ext(&self) -> Result { @@ -218,6 +202,67 @@ pub trait ActorType { } } +pub enum EndpointType { + Community, + User, + Post, + Comment, + PrivateMessage, +} + +/// Generates the ActivityPub ID for a given object type and ID. +pub fn generate_apub_endpoint( + endpoint_type: EndpointType, + name: &str, +) -> Result { + let point = match endpoint_type { + EndpointType::Community => "c", + EndpointType::User => "u", + EndpointType::Post => "post", + EndpointType::Comment => "comment", + EndpointType::PrivateMessage => "private_message", + }; + + Ok( + Url::parse(&format!( + "{}/{}/{}", + Settings::get().get_protocol_and_hostname(), + point, + name + ))? + .into(), + ) +} + +pub fn generate_followers_url( + actor_id: &lemmy_db_schema::Url, +) -> Result { + Ok(Url::parse(&format!("{}/followers", actor_id))?.into()) +} + +pub fn generate_inbox_url( + actor_id: &lemmy_db_schema::Url, +) -> Result { + Ok(Url::parse(&format!("{}/inbox", actor_id))?.into()) +} + +pub fn generate_shared_inbox_url( + actor_id: &lemmy_db_schema::Url, +) -> Result { + let actor_id = actor_id.clone().into_inner(); + let url = format!( + "{}://{}{}/inbox", + &actor_id.scheme(), + &actor_id.host_str().context(location_info!())?, + if let Some(port) = actor_id.port() { + format!(":{}", port) + } else { + "".to_string() + }, + ); + Ok(Url::parse(&url)?.into()) +} + /// Store a sent or received activity in the database, for logging purposes. These records are not /// persistent. pub(crate) async fn insert_activity( diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 566e4714..af947e04 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -83,13 +83,13 @@ impl ToApub for Community { group.set_image(image.into_any_base()?); } - let mut ap_actor = ApActor::new(self.get_inbox_url()?, group); + let mut ap_actor = ApActor::new(self.inbox_url.clone().into(), group); ap_actor .set_preferred_username(self.name.to_owned()) .set_outbox(self.get_outbox_url()?) - .set_followers(self.get_followers_url()?) + .set_followers(self.followers_url.clone().into()) .set_endpoints(Endpoints { - shared_inbox: Some(self.get_shared_inbox_url()?), + shared_inbox: Some(self.get_shared_inbox_or_inbox_url()), ..Default::default() }); @@ -184,7 +184,6 @@ impl FromApubToForm for CommunityForm { ), None => None, }; - let banner = match group.image() { Some(any_image) => Some( Image::from_any_base(any_image.as_one().context(location_info!())?.clone()) @@ -197,6 +196,12 @@ impl FromApubToForm for CommunityForm { ), None => None, }; + let shared_inbox = group + .inner + .endpoints()? + .map(|e| e.shared_inbox) + .flatten() + .map(|s| s.to_owned().into()); Ok(CommunityForm { name, @@ -216,6 +221,16 @@ impl FromApubToForm for CommunityForm { last_refreshed_at: Some(naive_now()), icon, banner, + followers_url: Some( + group + .inner + .followers()? + .context(location_info!())? + .to_owned() + .into(), + ), + inbox_url: Some(group.inner.inbox()?.to_owned().into()), + shared_inbox_url: Some(shared_inbox), }) } } diff --git a/crates/apub/src/objects/user.rs b/crates/apub/src/objects/user.rs index a0bd5173..8a911de2 100644 --- a/crates/apub/src/objects/user.rs +++ b/crates/apub/src/objects/user.rs @@ -71,12 +71,12 @@ impl ToApub for User_ { person.set_name(i); } - let mut ap_actor = ApActor::new(self.get_inbox_url()?, person); + let mut ap_actor = ApActor::new(self.inbox_url.clone().into(), person); ap_actor .set_preferred_username(self.name.to_owned()) .set_outbox(self.get_outbox_url()?) .set_endpoints(Endpoints { - shared_inbox: Some(self.get_shared_inbox_url()?), + shared_inbox: Some(self.get_shared_inbox_or_inbox_url()), ..Default::default() }); @@ -158,8 +158,13 @@ impl FromApubToForm for UserForm { .flatten() .map(|n| n.to_owned().xsd_string()) .flatten(); - let bio = get_source_markdown_value(person)?; + let shared_inbox = person + .inner + .endpoints()? + .map(|e| e.shared_inbox) + .flatten() + .map(|s| s.to_owned().into()); check_slurs(&name)?; check_slurs_opt(&preferred_username)?; @@ -190,6 +195,8 @@ impl FromApubToForm for UserForm { private_key: None, public_key: Some(person.ext_one.public_key.to_owned().public_key_pem), last_refreshed_at: Some(naive_now()), + inbox_url: Some(person.inner.inbox()?.to_owned().into()), + shared_inbox_url: Some(shared_inbox), }) } } diff --git a/crates/db_queries/src/aggregates/comment_aggregates.rs b/crates/db_queries/src/aggregates/comment_aggregates.rs index cab81261..61984727 100644 --- a/crates/db_queries/src/aggregates/comment_aggregates.rs +++ b/crates/db_queries/src/aggregates/comment_aggregates.rs @@ -67,6 +67,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -96,6 +98,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let another_inserted_user = User_::create(&conn, &another_user).unwrap(); @@ -118,6 +122,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_queries/src/aggregates/community_aggregates.rs b/crates/db_queries/src/aggregates/community_aggregates.rs index 0f15453a..baeaa759 100644 --- a/crates/db_queries/src/aggregates/community_aggregates.rs +++ b/crates/db_queries/src/aggregates/community_aggregates.rs @@ -71,6 +71,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -100,6 +102,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let another_inserted_user = User_::create(&conn, &another_user).unwrap(); @@ -122,6 +126,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); @@ -144,6 +151,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let another_inserted_community = Community::create(&conn, &another_community).unwrap(); diff --git a/crates/db_queries/src/aggregates/post_aggregates.rs b/crates/db_queries/src/aggregates/post_aggregates.rs index d5f78bf1..1d69fb40 100644 --- a/crates/db_queries/src/aggregates/post_aggregates.rs +++ b/crates/db_queries/src/aggregates/post_aggregates.rs @@ -70,6 +70,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -99,6 +101,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let another_inserted_user = User_::create(&conn, &another_user).unwrap(); @@ -121,6 +125,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_queries/src/aggregates/site_aggregates.rs b/crates/db_queries/src/aggregates/site_aggregates.rs index ce9f2f76..12551365 100644 --- a/crates/db_queries/src/aggregates/site_aggregates.rs +++ b/crates/db_queries/src/aggregates/site_aggregates.rs @@ -69,6 +69,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -105,6 +107,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_queries/src/aggregates/user_aggregates.rs b/crates/db_queries/src/aggregates/user_aggregates.rs index f1170456..8d232268 100644 --- a/crates/db_queries/src/aggregates/user_aggregates.rs +++ b/crates/db_queries/src/aggregates/user_aggregates.rs @@ -67,6 +67,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -96,6 +98,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let another_inserted_user = User_::create(&conn, &another_user).unwrap(); @@ -118,6 +122,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_queries/src/source/activity.rs b/crates/db_queries/src/source/activity.rs index d47bc256..b32c8f98 100644 --- a/crates/db_queries/src/source/activity.rs +++ b/crates/db_queries/src/source/activity.rs @@ -162,6 +162,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_creator = User_::create(&conn, &creator_form).unwrap(); diff --git a/crates/db_queries/src/source/comment.rs b/crates/db_queries/src/source/comment.rs index 28d52e89..3abff1c5 100644 --- a/crates/db_queries/src/source/comment.rs +++ b/crates/db_queries/src/source/comment.rs @@ -14,7 +14,7 @@ use lemmy_db_schema::{ }; pub trait Comment_ { - fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: String) -> Result; + fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: Url) -> Result; fn permadelete_for_creator( conn: &PgConnection, for_creator_id: i32, @@ -43,7 +43,7 @@ pub trait Comment_ { } impl Comment_ for Comment { - fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: String) -> Result { + fn update_ap_id(conn: &PgConnection, comment_id: i32, apub_id: Url) -> Result { use lemmy_db_schema::schema::comment::dsl::*; diesel::update(comment.find(comment_id)) @@ -242,6 +242,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -264,6 +266,9 @@ mod tests { published: None, banner: None, icon: None, + inbox_url: None, + shared_inbox_url: None, + followers_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_queries/src/source/community.rs b/crates/db_queries/src/source/community.rs index bb7a9c24..c2809a78 100644 --- a/crates/db_queries/src/source/community.rs +++ b/crates/db_queries/src/source/community.rs @@ -133,6 +133,7 @@ pub trait Community_ { new_creator_id: i32, ) -> Result; fn distinct_federated_communities(conn: &PgConnection) -> Result, Error>; + fn read_from_followers_url(conn: &PgConnection, followers_url: &Url) -> Result; } impl Community_ for Community { @@ -192,6 +193,16 @@ impl Community_ for Community { use lemmy_db_schema::schema::community::dsl::*; community.select(actor_id).distinct().load::(conn) } + + fn read_from_followers_url( + conn: &PgConnection, + followers_url_: &Url, + ) -> Result { + use lemmy_db_schema::schema::community::dsl::*; + community + .filter(followers_url.eq(followers_url_)) + .first::(conn) + } } impl Joinable for CommunityModerator { @@ -361,6 +372,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -383,6 +396,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); @@ -406,6 +422,9 @@ mod tests { last_refreshed_at: inserted_community.published, icon: None, banner: None, + followers_url: inserted_community.followers_url.to_owned(), + inbox_url: inserted_community.inbox_url.to_owned(), + shared_inbox_url: None, }; let community_follower_form = CommunityFollowerForm { diff --git a/crates/db_queries/src/source/moderator.rs b/crates/db_queries/src/source/moderator.rs index 93c42416..e3ee7632 100644 --- a/crates/db_queries/src/source/moderator.rs +++ b/crates/db_queries/src/source/moderator.rs @@ -230,6 +230,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_mod = User_::create(&conn, &new_mod).unwrap(); @@ -259,6 +261,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -281,6 +285,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_queries/src/source/password_reset_request.rs b/crates/db_queries/src/source/password_reset_request.rs index d4ba2f12..f58d2c01 100644 --- a/crates/db_queries/src/source/password_reset_request.rs +++ b/crates/db_queries/src/source/password_reset_request.rs @@ -110,6 +110,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); diff --git a/crates/db_queries/src/source/post.rs b/crates/db_queries/src/source/post.rs index 7816d4a1..ddb3702c 100644 --- a/crates/db_queries/src/source/post.rs +++ b/crates/db_queries/src/source/post.rs @@ -42,7 +42,7 @@ impl Crud for Post { pub trait Post_ { //fn read(conn: &PgConnection, post_id: i32) -> Result; fn list_for_community(conn: &PgConnection, the_community_id: i32) -> Result, Error>; - fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: String) -> Result; + fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: Url) -> Result; fn permadelete_for_creator(conn: &PgConnection, for_creator_id: i32) -> Result, Error>; fn update_deleted(conn: &PgConnection, post_id: i32, new_deleted: bool) -> Result; fn update_removed(conn: &PgConnection, post_id: i32, new_removed: bool) -> Result; @@ -68,7 +68,7 @@ impl Post_ for Post { .load::(conn) } - fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: String) -> Result { + fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: Url) -> Result { use lemmy_db_schema::schema::post::dsl::*; diesel::update(post.find(post_id)) @@ -261,6 +261,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -283,6 +285,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_queries/src/source/private_message.rs b/crates/db_queries/src/source/private_message.rs index 4e0f66b6..8cace938 100644 --- a/crates/db_queries/src/source/private_message.rs +++ b/crates/db_queries/src/source/private_message.rs @@ -53,7 +53,7 @@ pub trait PrivateMessage_ { fn update_ap_id( conn: &PgConnection, private_message_id: i32, - apub_id: String, + apub_id: Url, ) -> Result; fn update_content( conn: &PgConnection, @@ -80,7 +80,7 @@ impl PrivateMessage_ for PrivateMessage { fn update_ap_id( conn: &PgConnection, private_message_id: i32, - apub_id: String, + apub_id: Url, ) -> Result { use lemmy_db_schema::schema::private_message::dsl::*; @@ -177,6 +177,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_creator = User_::create(&conn, &creator_form).unwrap(); @@ -206,6 +208,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_recipient = User_::create(&conn, &recipient_form).unwrap(); diff --git a/crates/db_queries/src/source/user.rs b/crates/db_queries/src/source/user.rs index b7db3e2d..5f3fa6cb 100644 --- a/crates/db_queries/src/source/user.rs +++ b/crates/db_queries/src/source/user.rs @@ -28,6 +28,8 @@ mod safe_type { local, banner, deleted, + inbox_url, + shared_inbox_url, ); impl ToSafe for User_ { @@ -48,6 +50,8 @@ mod safe_type { local, banner, deleted, + inbox_url, + shared_inbox_url, ) } } @@ -405,6 +409,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -436,6 +442,8 @@ mod tests { public_key: None, last_refreshed_at: inserted_user.published, deleted: false, + inbox_url: inserted_user.inbox_url.to_owned(), + shared_inbox_url: None, }; let read_user = User_::read(&conn, inserted_user.id).unwrap(); diff --git a/crates/db_queries/src/source/user_mention.rs b/crates/db_queries/src/source/user_mention.rs index d9e0cce3..93f0b86d 100644 --- a/crates/db_queries/src/source/user_mention.rs +++ b/crates/db_queries/src/source/user_mention.rs @@ -111,6 +111,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -140,6 +142,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_recipient = User_::create(&conn, &recipient_form).unwrap(); @@ -162,6 +166,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 9ff73ecc..e808d770 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -99,6 +99,9 @@ table! { last_refreshed_at -> Timestamp, icon -> Nullable, banner -> Nullable, + followers_url -> Text, + inbox_url -> Text, + shared_inbox_url -> Nullable, } } @@ -410,6 +413,8 @@ table! { last_refreshed_at -> Timestamp, banner -> Nullable, deleted -> Bool, + inbox_url -> Text, + shared_inbox_url -> Nullable, } } diff --git a/crates/db_schema/src/source/community.rs b/crates/db_schema/src/source/community.rs index d938d265..e05abf1b 100644 --- a/crates/db_schema/src/source/community.rs +++ b/crates/db_schema/src/source/community.rs @@ -25,6 +25,9 @@ pub struct Community { pub last_refreshed_at: chrono::NaiveDateTime, pub icon: Option, pub banner: Option, + pub followers_url: Url, + pub inbox_url: Url, + pub shared_inbox_url: Option, } /// A safe representation of community, without the sensitive info @@ -68,6 +71,9 @@ pub struct CommunityForm { pub last_refreshed_at: Option, pub icon: Option>, pub banner: Option>, + pub followers_url: Option, + pub inbox_url: Option, + pub shared_inbox_url: Option>, } #[derive(Identifiable, Queryable, Associations, PartialEq, Debug)] diff --git a/crates/db_schema/src/source/user.rs b/crates/db_schema/src/source/user.rs index f702c84b..d72929fa 100644 --- a/crates/db_schema/src/source/user.rs +++ b/crates/db_schema/src/source/user.rs @@ -33,6 +33,8 @@ pub struct User_ { pub last_refreshed_at: chrono::NaiveDateTime, pub banner: Option, pub deleted: bool, + pub inbox_url: Url, + pub shared_inbox_url: Option, } /// A safe representation of user, without the sensitive info @@ -53,6 +55,8 @@ pub struct UserSafe { pub local: bool, pub banner: Option, pub deleted: bool, + pub inbox_url: Url, + pub shared_inbox_url: Option, } /// A safe user view with only settings @@ -211,4 +215,6 @@ pub struct UserForm { pub public_key: Option, pub last_refreshed_at: Option, pub banner: Option>, + pub inbox_url: Option, + pub shared_inbox_url: Option>, } diff --git a/crates/db_views/Cargo.toml b/crates/db_views/Cargo.toml index 70be2fb5..175e6eae 100644 --- a/crates/db_views/Cargo.toml +++ b/crates/db_views/Cargo.toml @@ -9,3 +9,4 @@ lemmy_db_schema = { path = "../db_schema" } diesel = { version = "1.4.5", features = ["postgres","chrono","r2d2","serde_json"] } serde = { version = "1.0.123", features = ["derive"] } log = "0.4.14" +url = "2.2.0" diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index 68eb0688..a262b7c1 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -471,6 +471,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -493,6 +495,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); @@ -581,6 +586,8 @@ mod tests { admin: false, updated: None, matrix_user_id: None, + inbox_url: inserted_user.inbox_url.to_owned(), + shared_inbox_url: None, }, recipient: None, post: Post { diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index d454fcbe..29e4c357 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -463,6 +463,8 @@ mod tests { private_key: None, public_key: None, last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_user = User_::create(&conn, &new_user).unwrap(); @@ -485,6 +487,9 @@ mod tests { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; let inserted_community = Community::create(&conn, &new_community).unwrap(); @@ -588,6 +593,8 @@ mod tests { admin: false, updated: None, matrix_user_id: None, + inbox_url: inserted_user.inbox_url.to_owned(), + shared_inbox_url: None, }, creator_banned_from_community: false, community: CommunitySafe { diff --git a/crates/utils/src/apub.rs b/crates/utils/src/apub.rs index 4f6ec22f..130ee26e 100644 --- a/crates/utils/src/apub.rs +++ b/crates/utils/src/apub.rs @@ -1,7 +1,5 @@ -use crate::settings::Settings; use openssl::{pkey::PKey, rsa::Rsa}; use std::io::{Error, ErrorKind}; -use url::Url; pub struct Keypair { pub private_key: String, @@ -26,30 +24,3 @@ pub fn generate_actor_keypair() -> Result { public_key: key_to_string(public_key)?, }) } - -pub enum EndpointType { - Community, - User, - Post, - Comment, - PrivateMessage, -} - -/// Generates the ActivityPub ID for a given object type and ID. -pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url { - let point = match endpoint_type { - EndpointType::Community => "c", - EndpointType::User => "u", - EndpointType::Post => "post", - EndpointType::Comment => "comment", - EndpointType::PrivateMessage => "private_message", - }; - - Url::parse(&format!( - "{}/{}/{}", - Settings::get().get_protocol_and_hostname(), - point, - name - )) - .unwrap() -} diff --git a/lemmy_db/src/schema.rs b/lemmy_db/src/schema.rs new file mode 100644 index 00000000..7bc66e2c --- /dev/null +++ b/lemmy_db/src/schema.rs @@ -0,0 +1,526 @@ +table! { + activity (id) { + id -> Int4, + data -> Jsonb, + local -> Bool, + published -> Timestamp, + updated -> Nullable, + ap_id -> Nullable, + sensitive -> Nullable, + } +} + +table! { + category (id) { + id -> Int4, + name -> Varchar, + } +} + +table! { + comment (id) { + id -> Int4, + creator_id -> Int4, + post_id -> Int4, + parent_id -> Nullable, + content -> Text, + removed -> Bool, + read -> Bool, + published -> Timestamp, + updated -> Nullable, + deleted -> Bool, + ap_id -> Varchar, + local -> Bool, + } +} + +table! { + comment_aggregates (id) { + id -> Int4, + comment_id -> Int4, + score -> Int8, + upvotes -> Int8, + downvotes -> Int8, + published -> Timestamp, + } +} + +table! { + comment_like (id) { + id -> Int4, + user_id -> Int4, + comment_id -> Int4, + post_id -> Int4, + score -> Int2, + published -> Timestamp, + } +} + +table! { + comment_report (id) { + id -> Int4, + creator_id -> Int4, + comment_id -> Int4, + original_comment_text -> Text, + reason -> Text, + resolved -> Bool, + resolver_id -> Nullable, + published -> Timestamp, + updated -> Nullable, + } +} + +table! { + comment_saved (id) { + id -> Int4, + comment_id -> Int4, + user_id -> Int4, + published -> Timestamp, + } +} + +table! { + community (id) { + id -> Int4, + name -> Varchar, + title -> Varchar, + description -> Nullable, + category_id -> Int4, + creator_id -> Int4, + removed -> Bool, + published -> Timestamp, + updated -> Nullable, + deleted -> Bool, + nsfw -> Bool, + actor_id -> Varchar, + local -> Bool, + private_key -> Nullable, + public_key -> Nullable, + last_refreshed_at -> Timestamp, + icon -> Nullable, + banner -> Nullable, + followers_url -> Text, + inbox_url -> Text, + shared_inbox_url -> Nullable, + } +} + +table! { + community_aggregates (id) { + id -> Int4, + community_id -> Int4, + subscribers -> Int8, + posts -> Int8, + comments -> Int8, + published -> Timestamp, + } +} + +table! { + community_follower (id) { + id -> Int4, + community_id -> Int4, + user_id -> Int4, + published -> Timestamp, + pending -> Nullable, + } +} + +table! { + community_moderator (id) { + id -> Int4, + community_id -> Int4, + user_id -> Int4, + published -> Timestamp, + } +} + +table! { + community_user_ban (id) { + id -> Int4, + community_id -> Int4, + user_id -> Int4, + published -> Timestamp, + } +} + +table! { + mod_add (id) { + id -> Int4, + mod_user_id -> Int4, + other_user_id -> Int4, + removed -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_add_community (id) { + id -> Int4, + mod_user_id -> Int4, + other_user_id -> Int4, + community_id -> Int4, + removed -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_ban (id) { + id -> Int4, + mod_user_id -> Int4, + other_user_id -> Int4, + reason -> Nullable, + banned -> Nullable, + expires -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_ban_from_community (id) { + id -> Int4, + mod_user_id -> Int4, + other_user_id -> Int4, + community_id -> Int4, + reason -> Nullable, + banned -> Nullable, + expires -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_lock_post (id) { + id -> Int4, + mod_user_id -> Int4, + post_id -> Int4, + locked -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_remove_comment (id) { + id -> Int4, + mod_user_id -> Int4, + comment_id -> Int4, + reason -> Nullable, + removed -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_remove_community (id) { + id -> Int4, + mod_user_id -> Int4, + community_id -> Int4, + reason -> Nullable, + removed -> Nullable, + expires -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_remove_post (id) { + id -> Int4, + mod_user_id -> Int4, + post_id -> Int4, + reason -> Nullable, + removed -> Nullable, + when_ -> Timestamp, + } +} + +table! { + mod_sticky_post (id) { + id -> Int4, + mod_user_id -> Int4, + post_id -> Int4, + stickied -> Nullable, + when_ -> Timestamp, + } +} + +table! { + password_reset_request (id) { + id -> Int4, + user_id -> Int4, + token_encrypted -> Text, + published -> Timestamp, + } +} + +table! { + post (id) { + id -> Int4, + name -> Varchar, + url -> Nullable, + body -> Nullable, + creator_id -> Int4, + community_id -> Int4, + removed -> Bool, + locked -> Bool, + published -> Timestamp, + updated -> Nullable, + deleted -> Bool, + nsfw -> Bool, + stickied -> Bool, + embed_title -> Nullable, + embed_description -> Nullable, + embed_html -> Nullable, + thumbnail_url -> Nullable, + ap_id -> Varchar, + local -> Bool, + } +} + +table! { + post_aggregates (id) { + id -> Int4, + post_id -> Int4, + comments -> Int8, + score -> Int8, + upvotes -> Int8, + downvotes -> Int8, + stickied -> Bool, + published -> Timestamp, + newest_comment_time -> Timestamp, + } +} + +table! { + post_like (id) { + id -> Int4, + post_id -> Int4, + user_id -> Int4, + score -> Int2, + published -> Timestamp, + } +} + +table! { + post_read (id) { + id -> Int4, + post_id -> Int4, + user_id -> Int4, + published -> Timestamp, + } +} + +table! { + post_report (id) { + id -> Int4, + creator_id -> Int4, + post_id -> Int4, + original_post_name -> Varchar, + original_post_url -> Nullable, + original_post_body -> Nullable, + reason -> Text, + resolved -> Bool, + resolver_id -> Nullable, + published -> Timestamp, + updated -> Nullable, + } +} + +table! { + post_saved (id) { + id -> Int4, + post_id -> Int4, + user_id -> Int4, + published -> Timestamp, + } +} + +table! { + private_message (id) { + id -> Int4, + creator_id -> Int4, + recipient_id -> Int4, + content -> Text, + deleted -> Bool, + read -> Bool, + published -> Timestamp, + updated -> Nullable, + ap_id -> Varchar, + local -> Bool, + } +} + +table! { + site (id) { + id -> Int4, + name -> Varchar, + description -> Nullable, + creator_id -> Int4, + published -> Timestamp, + updated -> Nullable, + enable_downvotes -> Bool, + open_registration -> Bool, + enable_nsfw -> Bool, + icon -> Nullable, + banner -> Nullable, + } +} + +table! { + site_aggregates (id) { + id -> Int4, + site_id -> Int4, + users -> Int8, + posts -> Int8, + comments -> Int8, + communities -> Int8, + } +} + +table! { + user_ (id) { + id -> Int4, + name -> Varchar, + preferred_username -> Nullable, + password_encrypted -> Text, + email -> Nullable, + avatar -> Nullable, + admin -> Bool, + banned -> Bool, + published -> Timestamp, + updated -> Nullable, + show_nsfw -> Bool, + theme -> Varchar, + default_sort_type -> Int2, + default_listing_type -> Int2, + lang -> Varchar, + show_avatars -> Bool, + send_notifications_to_email -> Bool, + matrix_user_id -> Nullable, + actor_id -> Varchar, + bio -> Nullable, + local -> Bool, + private_key -> Nullable, + public_key -> Nullable, + last_refreshed_at -> Timestamp, + banner -> Nullable, + deleted -> Bool, + inbox_url -> Text, + shared_inbox_url -> Nullable, + } +} + +table! { + user_aggregates (id) { + id -> Int4, + user_id -> Int4, + post_count -> Int8, + post_score -> Int8, + comment_count -> Int8, + comment_score -> Int8, + } +} + +table! { + user_ban (id) { + id -> Int4, + user_id -> Int4, + published -> Timestamp, + } +} + +table! { + user_mention (id) { + id -> Int4, + recipient_id -> Int4, + comment_id -> Int4, + read -> Bool, + published -> Timestamp, + } +} + +joinable!(comment -> post (post_id)); +joinable!(comment -> user_ (creator_id)); +joinable!(comment_aggregates -> comment (comment_id)); +joinable!(comment_like -> comment (comment_id)); +joinable!(comment_like -> post (post_id)); +joinable!(comment_like -> user_ (user_id)); +joinable!(comment_report -> comment (comment_id)); +joinable!(comment_saved -> comment (comment_id)); +joinable!(comment_saved -> user_ (user_id)); +joinable!(community -> category (category_id)); +joinable!(community -> user_ (creator_id)); +joinable!(community_aggregates -> community (community_id)); +joinable!(community_follower -> community (community_id)); +joinable!(community_follower -> user_ (user_id)); +joinable!(community_moderator -> community (community_id)); +joinable!(community_moderator -> user_ (user_id)); +joinable!(community_user_ban -> community (community_id)); +joinable!(community_user_ban -> user_ (user_id)); +joinable!(mod_add_community -> community (community_id)); +joinable!(mod_ban_from_community -> community (community_id)); +joinable!(mod_lock_post -> post (post_id)); +joinable!(mod_lock_post -> user_ (mod_user_id)); +joinable!(mod_remove_comment -> comment (comment_id)); +joinable!(mod_remove_comment -> user_ (mod_user_id)); +joinable!(mod_remove_community -> community (community_id)); +joinable!(mod_remove_community -> user_ (mod_user_id)); +joinable!(mod_remove_post -> post (post_id)); +joinable!(mod_remove_post -> user_ (mod_user_id)); +joinable!(mod_sticky_post -> post (post_id)); +joinable!(mod_sticky_post -> user_ (mod_user_id)); +joinable!(password_reset_request -> user_ (user_id)); +joinable!(post -> community (community_id)); +joinable!(post -> user_ (creator_id)); +joinable!(post_aggregates -> post (post_id)); +joinable!(post_like -> post (post_id)); +joinable!(post_like -> user_ (user_id)); +joinable!(post_read -> post (post_id)); +joinable!(post_read -> user_ (user_id)); +joinable!(post_report -> post (post_id)); +joinable!(post_saved -> post (post_id)); +joinable!(post_saved -> user_ (user_id)); +joinable!(site -> user_ (creator_id)); +joinable!(site_aggregates -> site (site_id)); +joinable!(user_aggregates -> user_ (user_id)); +joinable!(user_ban -> user_ (user_id)); +joinable!(user_mention -> comment (comment_id)); +joinable!(user_mention -> user_ (recipient_id)); + +allow_tables_to_appear_in_same_query!( + activity, + category, + comment, + comment_aggregates, + comment_like, + comment_report, + comment_saved, + community, + community_aggregates, + community_follower, + community_moderator, + community_user_ban, + mod_add, + mod_add_community, + mod_ban, + mod_ban_from_community, + mod_lock_post, + mod_remove_comment, + mod_remove_community, + mod_remove_post, + mod_sticky_post, + password_reset_request, + post, + post_aggregates, + post_like, + post_read, + post_report, + post_saved, + private_message, + site, + site_aggregates, + user_, + user_aggregates, + user_ban, + user_mention, +); diff --git a/migrations/2021-02-02-153240_apub_columns/down.sql b/migrations/2021-02-02-153240_apub_columns/down.sql new file mode 100644 index 00000000..248eb9b8 --- /dev/null +++ b/migrations/2021-02-02-153240_apub_columns/down.sql @@ -0,0 +1,6 @@ +ALTER TABLE community DROP COLUMN followers_url; +ALTER TABLE community DROP COLUMN inbox_url; +ALTER TABLE community DROP COLUMN shared_inbox_url; + +ALTER TABLE user_ DROP COLUMN inbox_url; +ALTER TABLE user_ DROP COLUMN shared_inbox_url; \ No newline at end of file diff --git a/migrations/2021-02-02-153240_apub_columns/up.sql b/migrations/2021-02-02-153240_apub_columns/up.sql new file mode 100644 index 00000000..48f3b20e --- /dev/null +++ b/migrations/2021-02-02-153240_apub_columns/up.sql @@ -0,0 +1,10 @@ +ALTER TABLE community ADD COLUMN followers_url varchar(255) NOT NULL DEFAULT generate_unique_changeme(); +ALTER TABLE community ADD COLUMN inbox_url varchar(255) NOT NULL DEFAULT generate_unique_changeme(); +ALTER TABLE community ADD COLUMN shared_inbox_url varchar(255); + +ALTER TABLE user_ ADD COLUMN inbox_url varchar(255) NOT NULL DEFAULT generate_unique_changeme(); +ALTER TABLE user_ ADD COLUMN shared_inbox_url varchar(255); + +ALTER TABLE community ADD CONSTRAINT idx_community_followers_url UNIQUE (followers_url); +ALTER TABLE community ADD CONSTRAINT idx_community_inbox_url UNIQUE (inbox_url); +ALTER TABLE user_ ADD CONSTRAINT idx_user_inbox_url UNIQUE (inbox_url); diff --git a/scripts/test.sh b/scripts/test.sh index 21093d0c..b47f09fa 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,6 +1,9 @@ #!/bin/sh set -e +psql -U lemmy -d postgres -c "DROP DATABASE lemmy;" +psql -U lemmy -d postgres -c "CREATE DATABASE lemmy;" + export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy # Commenting since this will overwrite schema.rs, which will break things now # diesel migration run diff --git a/src/code_migrations.rs b/src/code_migrations.rs index a6586ae7..e8daf56e 100644 --- a/src/code_migrations.rs +++ b/src/code_migrations.rs @@ -3,6 +3,13 @@ use diesel::{ sql_types::{Nullable, Text}, *, }; +use lemmy_apub::{ + generate_apub_endpoint, + generate_followers_url, + generate_inbox_url, + generate_shared_inbox_url, + EndpointType, +}; use lemmy_db_queries::{ source::{comment::Comment_, post::Post_, private_message::PrivateMessage_}, Crud, @@ -17,11 +24,7 @@ use lemmy_db_schema::{ user::{UserForm, User_}, }, }; -use lemmy_utils::{ - apub::{generate_actor_keypair, make_apub_endpoint, EndpointType}, - settings::Settings, - LemmyError, -}; +use lemmy_utils::{apub::generate_actor_keypair, settings::Settings, LemmyError}; use log::info; pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), LemmyError> { @@ -31,6 +34,7 @@ pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), LemmyError> { comment_updates_2020_04_03(&conn)?; private_message_updates_2020_05_05(&conn)?; post_thumbnail_url_updates_2020_07_27(&conn)?; + apub_columns_2021_02_02(&conn)?; Ok(()) } @@ -68,12 +72,14 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { lang: cuser.lang.to_owned(), show_avatars: cuser.show_avatars, send_notifications_to_email: cuser.send_notifications_to_email, - actor_id: Some(make_apub_endpoint(EndpointType::User, &cuser.name).into()), + actor_id: Some(generate_apub_endpoint(EndpointType::User, &cuser.name)?), bio: Some(cuser.bio.to_owned()), local: cuser.local, private_key: Some(keypair.private_key), public_key: Some(keypair.public_key), last_refreshed_at: Some(naive_now()), + inbox_url: None, + shared_inbox_url: None, }; User_::update(&conn, cuser.id, &form)?; @@ -97,6 +103,7 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { for ccommunity in &incorrect_communities { let keypair = generate_actor_keypair()?; + let community_actor_id = generate_apub_endpoint(EndpointType::Community, &ccommunity.name)?; let form = CommunityForm { name: ccommunity.name.to_owned(), @@ -108,7 +115,7 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { deleted: None, nsfw: ccommunity.nsfw, updated: None, - actor_id: Some(make_apub_endpoint(EndpointType::Community, &ccommunity.name).into()), + actor_id: Some(community_actor_id.to_owned()), local: ccommunity.local, private_key: Some(keypair.private_key), public_key: Some(keypair.public_key), @@ -116,6 +123,9 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { published: None, icon: Some(ccommunity.icon.to_owned()), banner: Some(ccommunity.banner.to_owned()), + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; Community::update(&conn, ccommunity.id, &form)?; @@ -138,7 +148,7 @@ fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { .load::(conn)?; for cpost in &incorrect_posts { - let apub_id = make_apub_endpoint(EndpointType::Post, &cpost.id.to_string()).to_string(); + let apub_id = generate_apub_endpoint(EndpointType::Post, &cpost.id.to_string())?; Post::update_ap_id(&conn, cpost.id, apub_id)?; } @@ -159,7 +169,7 @@ fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), LemmyError> { .load::(conn)?; for ccomment in &incorrect_comments { - let apub_id = make_apub_endpoint(EndpointType::Comment, &ccomment.id.to_string()).to_string(); + let apub_id = generate_apub_endpoint(EndpointType::Comment, &ccomment.id.to_string())?; Comment::update_ap_id(&conn, ccomment.id, apub_id)?; } @@ -180,7 +190,7 @@ fn private_message_updates_2020_05_05(conn: &PgConnection) -> Result<(), LemmyEr .load::(conn)?; for cpm in &incorrect_pms { - let apub_id = make_apub_endpoint(EndpointType::PrivateMessage, &cpm.id.to_string()).to_string(); + let apub_id = generate_apub_endpoint(EndpointType::PrivateMessage, &cpm.id.to_string())?; PrivateMessage::update_ap_id(&conn, cpm.id, apub_id)?; } @@ -216,3 +226,48 @@ fn post_thumbnail_url_updates_2020_07_27(conn: &PgConnection) -> Result<(), Lemm Ok(()) } + +/// We are setting inbox and follower URLs for local and remote actors alike, because for now +/// all federated instances are also Lemmy and use the same URL scheme. +fn apub_columns_2021_02_02(conn: &PgConnection) -> Result<(), LemmyError> { + info!("Running apub_columns_2021_02_02"); + { + use lemmy_db_schema::schema::user_::dsl::*; + let users = user_ + .filter(inbox_url.eq("http://changeme_%")) + .load::(conn)?; + + for u in &users { + let inbox_url_ = generate_inbox_url(&u.actor_id)?; + let shared_inbox_url_ = generate_shared_inbox_url(&u.actor_id)?; + diesel::update(user_.find(u.id)) + .set(( + inbox_url.eq(inbox_url_), + shared_inbox_url.eq(shared_inbox_url_), + )) + .get_result::(conn)?; + } + } + + { + use lemmy_db_schema::schema::community::dsl::*; + let communities = community + .filter(inbox_url.eq("http://changeme_%")) + .load::(conn)?; + + for c in &communities { + let followers_url_ = generate_followers_url(&c.actor_id)?; + let inbox_url_ = generate_inbox_url(&c.actor_id)?; + let shared_inbox_url_ = generate_shared_inbox_url(&c.actor_id)?; + diesel::update(community.find(c.id)) + .set(( + followers_url.eq(followers_url_), + inbox_url.eq(inbox_url_), + shared_inbox_url.eq(shared_inbox_url_), + )) + .get_result::(conn)?; + } + } + + Ok(()) +} diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 28e67a71..f205ceb5 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -114,6 +114,8 @@ fn create_user(conn: &PgConnection, name: &str) -> User_ { private_key: Some(user_keypair.private_key), public_key: Some(user_keypair.public_key), last_refreshed_at: None, + inbox_url: None, + shared_inbox_url: None, }; User_::create(&conn, &new_user).unwrap() @@ -138,6 +140,9 @@ fn create_community(conn: &PgConnection, creator_id: i32) -> Community { published: None, icon: None, banner: None, + followers_url: None, + inbox_url: None, + shared_inbox_url: None, }; Community::create(&conn, &new_community).unwrap() }