From: Felix Ableitner Date: Fri, 3 Apr 2020 05:02:43 +0000 (+0200) Subject: Share list of communities over apub, some refactoring X-Git-Url: http://these/git/%7B%60%24%7BghostArchiveUrl%7D/static/%7Bpost_view.post.url%7D?a=commitdiff_plain;h=96c3621a801d96a8d2bffc849d871c297683e387;p=lemmy.git Share list of communities over apub, some refactoring --- diff --git a/server/Cargo.lock b/server/Cargo.lock index fb66be39..ee1b922b 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [[package]] name = "activitystreams" -version = "0.5.0-alpha.10" +version = "0.5.0-alpha.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "activitystreams-derive 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "activitystreams-derive 0.5.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", @@ -16,7 +16,7 @@ dependencies = [ [[package]] name = "activitystreams-derive" -version = "0.5.0-alpha.4" +version = "0.5.0-alpha.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1401,7 +1401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "lemmy_server" version = "0.0.1" dependencies = [ - "activitystreams 0.5.0-alpha.10 (registry+https://github.com/rust-lang/crates.io-index)", + "activitystreams 0.5.0-alpha.15 (registry+https://github.com/rust-lang/crates.io-index)", "actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "actix-files 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "actix-rt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2855,8 +2855,8 @@ dependencies = [ ] [metadata] -"checksum activitystreams 0.5.0-alpha.10 (registry+https://github.com/rust-lang/crates.io-index)" = "04827f3390831f772d15ff3171336cc4fbac714f59233d24731beb7c865293a6" -"checksum activitystreams-derive 0.5.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2bc640808dceb2efac81e6bcb77a7f4e2e76af7fb60e88f966b48123b625d2f" +"checksum activitystreams 0.5.0-alpha.15 (registry+https://github.com/rust-lang/crates.io-index)" = "1ecff5b18578fac8cff5f139009cd0a4f605b6f5eddc91f2b8319bfef76cc2c3" +"checksum activitystreams-derive 0.5.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d7498811842309cc5b54c123bcf328e33eebe9841037067b1e9b93caa820085d" "checksum actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4af87564ff659dee8f9981540cac9418c45e910c8072fdedd643a262a38fcaf" "checksum actix-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "09e55f0a5c2ca15795035d90c46bd0e73a5123b72f68f12596d6ba5282051380" "checksum actix-connect 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c95cc9569221e9802bf4c377f6c18b90ef10227d787611decf79fd47d2a8e76c" diff --git a/server/Cargo.toml b/server/Cargo.toml index c694c88c..548406fa 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -9,7 +9,7 @@ diesel = { version = "1.4.2", features = ["postgres","chrono", "r2d2", "64-colum diesel_migrations = "1.4.0" dotenv = "0.15.0" bcrypt = "0.6.1" -activitystreams = "0.5.0-alpha.10" +activitystreams = "0.5.0-alpha.15" chrono = { version = "0.4.7", features = ["serde"] } failure = "0.1.5" serde_json = { version = "1.0.45", features = ["preserve_order"]} diff --git a/server/src/api/community.rs b/server/src/api/community.rs index dac8733b..2bc6e357 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -36,11 +36,11 @@ pub struct CommunityResponse { #[derive(Serialize, Deserialize, Debug)] pub struct ListCommunities { - sort: String, - page: Option, - limit: Option, - auth: Option, - local_only: Option, + pub sort: String, + pub page: Option, + pub limit: Option, + pub auth: Option, + pub local_only: Option, } #[derive(Serialize, Deserialize, Debug)] diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index 2f533b97..67cd99c4 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -1,15 +1,19 @@ -use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType}; +use crate::api::community::ListCommunities; +use crate::api::{Oper, Perform}; +use crate::apub::puller::{fetch_remote_object, format_community_name}; +use crate::apub::{ + create_apub_response, get_apub_protocol_string, make_apub_endpoint, EndpointType, GroupExt, +}; use crate::convert_datetime; use crate::db::community::Community; -use crate::db::community_view::CommunityFollowerView; +use crate::db::community_view::{CommunityFollowerView, CommunityView}; use crate::db::establish_unpooled_connection; use crate::db::post_view::{PostQueryBuilder, PostView}; +use crate::settings::Settings; +use activitystreams::actor::properties::ApActorProperties; use activitystreams::collection::OrderedCollection; use activitystreams::{ - actor::{properties::ApActorProperties, Group}, - collection::UnorderedCollection, - context, - ext::Extensible, + actor::Group, collection::UnorderedCollection, context, ext::Extensible, object::properties::ObjectProperties, }; use actix_web::body::Body; @@ -26,41 +30,126 @@ pub struct CommunityQuery { community_name: String, } -pub async fn get_apub_community( - info: Path, +pub async fn get_apub_community_list( db: web::Data>>, ) -> Result, Error> { - let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; - let base_url = make_apub_endpoint(EndpointType::Community, &community.name); + // TODO: implement pagination + let query = ListCommunities { + sort: "Hot".to_string(), + page: None, + limit: None, + auth: None, + local_only: Some(true), + }; + let communities = Oper::new(query) + .perform(&db.get().unwrap()) + .unwrap() + .communities + .iter() + .map(|c| c.as_group()) + .collect::, failure::Error>>()?; + let mut collection = UnorderedCollection::default(); + let oprops: &mut ObjectProperties = collection.as_mut(); + oprops.set_context_xsd_any_uri(context())?.set_id(format!( + "{}://{}/federation/communities", + get_apub_protocol_string(), + Settings::get().hostname + ))?; - let mut group = Group::default(); - let oprops: &mut ObjectProperties = group.as_mut(); + collection + .collection_props + .set_total_items(communities.len() as u64)? + .set_many_items_base_boxes(communities)?; + Ok(create_apub_response(&collection)) +} - oprops - .set_context_xsd_any_uri(context())? - .set_id(base_url.to_owned())? - .set_name_xsd_string(community.title.to_owned())? - .set_published(convert_datetime(community.published))? - .set_attributed_to_xsd_any_uri(make_apub_endpoint( - EndpointType::User, - &community.creator_id.to_string(), - ))?; - - if let Some(u) = community.updated.to_owned() { - oprops.set_updated(convert_datetime(u))?; - } - if let Some(d) = community.description { - oprops.set_summary_xsd_string(d)?; - } +impl CommunityView { + fn as_group(&self) -> Result { + let base_url = make_apub_endpoint(EndpointType::Community, &self.name); + + let mut group = Group::default(); + let oprops: &mut ObjectProperties = group.as_mut(); + + oprops + .set_context_xsd_any_uri(context())? + .set_id(base_url.to_owned())? + .set_name_xsd_string(self.name.to_owned())? + .set_published(convert_datetime(self.published))? + .set_attributed_to_xsd_any_uri(make_apub_endpoint( + EndpointType::User, + &self.creator_id.to_string(), + ))?; + + if let Some(u) = self.updated.to_owned() { + oprops.set_updated(convert_datetime(u))?; + } + if let Some(d) = self.description.to_owned() { + oprops.set_summary_xsd_string(d)?; + } - let mut actor_props = ApActorProperties::default(); + let mut actor_props = ApActorProperties::default(); - actor_props - .set_inbox(format!("{}/inbox", &base_url))? - .set_outbox(format!("{}/outbox", &base_url))? - .set_followers(format!("{}/followers", &base_url))?; + actor_props + .set_preferred_username(self.title.to_owned())? + .set_inbox(format!("{}/inbox", &base_url))? + .set_outbox(format!("{}/outbox", &base_url))? + .set_followers(format!("{}/followers", &base_url))?; - Ok(create_apub_response(&group.extend(actor_props))) + Ok(group.extend(actor_props)) + } + + pub fn from_group(group: &GroupExt, domain: &str) -> Result { + let followers_uri = &group.extension.get_followers().unwrap().to_string(); + let outbox_uri = &group.extension.get_outbox().to_string(); + let outbox = fetch_remote_object::(outbox_uri)?; + let followers = fetch_remote_object::(followers_uri)?; + let oprops = &group.base.object_props; + let aprops = &group.extension; + Ok(CommunityView { + // TODO: we need to merge id and name into a single thing (stuff like @user@instance.com) + id: 1337, //community.object_props.get_id() + name: format_community_name(&oprops.get_name_xsd_string().unwrap().to_string(), domain), + title: aprops.get_preferred_username().unwrap().to_string(), + description: oprops.get_summary_xsd_string().map(|s| s.to_string()), + category_id: -1, + creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri() + removed: false, + published: oprops + .get_published() + .unwrap() + .as_ref() + .naive_local() + .to_owned(), + updated: oprops + .get_updated() + .map(|u| u.as_ref().to_owned().naive_local()), + deleted: false, + nsfw: false, + creator_name: "".to_string(), + creator_avatar: None, + category_name: "".to_string(), + number_of_subscribers: *followers + .collection_props + .get_total_items() + .unwrap() + .as_ref() as i64, + number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64, + number_of_comments: -1, + hot_rank: -1, + user_id: None, + subscribed: None, + }) + } +} + +pub async fn get_apub_community_http( + info: Path, + db: web::Data>>, +) -> Result, Error> { + let community = Community::read_from_name(&&db.get()?, info.community_name.to_owned())?; + let community_view = CommunityView::read(&&db.get()?, community.id, None)?; + let c = community_view.as_group()?; + Ok(create_apub_response(&c)) } pub async fn get_apub_community_followers( @@ -107,7 +196,7 @@ pub async fn get_apub_community_outbox( .set_id(base_url)?; collection .collection_props - .set_many_items_object_boxs( + .set_many_items_base_boxes( community_posts .iter() .map(|c| c.as_page().unwrap()) diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index ed6ac656..8e725805 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -4,10 +4,14 @@ pub mod puller; pub mod user; use crate::Settings; +use activitystreams::actor::{properties::ApActorProperties, Group}; +use activitystreams::ext::Ext; use actix_web::body::Body; use actix_web::HttpResponse; use url::Url; +type GroupExt = Ext; + fn create_apub_response(json: &T) -> HttpResponse where T: serde::ser::Serialize, diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index 706c1134..a9b50013 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -1,6 +1,6 @@ use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType}; -use crate::convert_datetime; use crate::db::post_view::PostView; +use crate::{convert_datetime, naive_now}; use activitystreams::{object::properties::ObjectProperties, object::Page}; use actix_web::body::Body; use actix_web::web::Path; @@ -59,4 +59,53 @@ impl PostView { Ok(page) } + + pub fn from_page(page: &Page) -> Result { + let oprops = &page.object_props; + Ok(PostView { + id: -1, + name: oprops.get_name_xsd_string().unwrap().to_string(), + url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()), + body: oprops.get_content_xsd_string().map(|c| c.to_string()), + creator_id: -1, + community_id: -1, + removed: false, + locked: false, + published: oprops + .get_published() + .unwrap() + .as_ref() + .naive_local() + .to_owned(), + updated: oprops + .get_updated() + .map(|u| u.as_ref().to_owned().naive_local()), + deleted: false, + nsfw: false, + stickied: false, + embed_title: None, + embed_description: None, + embed_html: None, + thumbnail_url: None, + banned: false, + banned_from_community: false, + creator_name: "".to_string(), + creator_avatar: None, + community_name: "".to_string(), + community_removed: false, + community_deleted: false, + community_nsfw: false, + number_of_comments: -1, + score: -1, + upvotes: -1, + downvotes: -1, + hot_rank: -1, + newest_activity_time: naive_now(), + user_id: None, + my_vote: None, + subscribed: None, + read: None, + saved: None, + }) + } } diff --git a/server/src/apub/puller.rs b/server/src/apub/puller.rs index 0dd7fca2..c92023ec 100644 --- a/server/src/apub/puller.rs +++ b/server/src/apub/puller.rs @@ -1,16 +1,13 @@ -use crate::api::community::{GetCommunityResponse, ListCommunitiesResponse}; +use crate::api::community::GetCommunityResponse; use crate::api::post::GetPostsResponse; -use crate::apub::get_apub_protocol_string; +use crate::apub::{get_apub_protocol_string, GroupExt}; use crate::db::community_view::CommunityView; use crate::db::post_view::PostView; -use crate::naive_now; use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; use crate::settings::Settings; -use activitystreams::actor::{properties::ApActorProperties, Group}; use activitystreams::collection::{OrderedCollection, UnorderedCollection}; -use activitystreams::ext::Ext; -use activitystreams::object::ObjectBox; use activitystreams::object::Page; +use activitystreams::BaseBox; use chttp::prelude::*; use failure::Error; use log::warn; @@ -28,39 +25,44 @@ fn fetch_node_info(domain: &str) -> Result { fn fetch_communities_from_instance(domain: &str) -> Result, Error> { let node_info = fetch_node_info(domain)?; - if node_info.software.name != "lemmy" { - return Err(format_err!( + + if let Some(community_list_url) = node_info.metadata.community_list_url { + let collection = fetch_remote_object::(&community_list_url)?; + let object_boxes = collection + .collection_props + .get_many_items_base_boxes() + .unwrap(); + let communities: Result, Error> = object_boxes + .map(|c| -> Result { + let group = c.to_owned().to_concrete::()?; + CommunityView::from_group(&group, domain) + }) + .collect(); + Ok(communities?) + } else { + Err(format_err!( "{} is not a Lemmy instance, federation is not supported", domain - )); + )) } +} - // TODO: follow pagination (seems like page count is missing?) - // TODO: see if there is any standard for discovering remote actors, so we dont have to rely on lemmy apis - let communities_uri = format!( - "http://{}/api/v1/communities/list?sort=Hot&local_only=true", - domain - ); - let communities1 = fetch_remote_object::(&communities_uri)?; - let mut communities2 = communities1.communities; - for c in &mut communities2 { - c.name = format_community_name(&c.name, domain); - } - Ok(communities2) +/// Returns a tuple of (username, domain) from an identifier like "main@dev.lemmy.ml" +fn split_identifier(identifier: &str) -> (String, String) { + let x: Vec<&str> = identifier.split('@').collect(); + (x[0].replace("!", ""), x[1].to_string()) } fn get_remote_community_uri(identifier: &str) -> String { - let x: Vec<&str> = identifier.split('@').collect(); - let name = x[0].replace("!", ""); - let instance = x[1]; - format!("http://{}/federation/c/{}", instance, name) + let (name, domain) = split_identifier(identifier); + format!("http://{}/federation/c/{}", domain, name) } -fn fetch_remote_object(uri: &str) -> Result +pub fn fetch_remote_object(uri: &str) -> Result where Response: for<'de> Deserialize<'de>, { - if Settings::get().federation.tls_enabled && !uri.starts_with("https") { + if Settings::get().federation.tls_enabled && !uri.starts_with("https://") { return Err(format_err!("Activitypub uri is insecure: {}", uri)); } // TODO: should cache responses here when we are in production @@ -71,131 +73,32 @@ where } pub fn get_remote_community_posts(identifier: &str) -> Result { - let community = - fetch_remote_object::>(&get_remote_community_uri(identifier))?; + let community = fetch_remote_object::(&get_remote_community_uri(identifier))?; let outbox_uri = &community.extension.get_outbox().to_string(); let outbox = fetch_remote_object::(outbox_uri)?; - let items = outbox.collection_props.get_many_items_object_boxs(); + let items = outbox.collection_props.get_many_items_base_boxes(); - let posts: Vec = items + let posts: Result, Error> = items .unwrap() - .map(|obox: &ObjectBox| { - let page: Page = obox.clone().to_concrete::().unwrap(); - PostView { - id: -1, - name: page.object_props.get_name_xsd_string().unwrap().to_string(), - url: page - .object_props - .get_url_xsd_any_uri() - .map(|u| u.to_string()), - body: page - .object_props - .get_content_xsd_string() - .map(|c| c.to_string()), - creator_id: -1, - community_id: -1, - removed: false, - locked: false, - published: page - .object_props - .get_published() - .unwrap() - .as_ref() - .naive_local() - .to_owned(), - updated: page - .object_props - .get_updated() - .map(|u| u.as_ref().to_owned().naive_local()), - deleted: false, - nsfw: false, - stickied: false, - embed_title: None, - embed_description: None, - embed_html: None, - thumbnail_url: None, - banned: false, - banned_from_community: false, - creator_name: "".to_string(), - creator_avatar: None, - community_name: "".to_string(), - community_removed: false, - community_deleted: false, - community_nsfw: false, - number_of_comments: -1, - score: -1, - upvotes: -1, - downvotes: -1, - hot_rank: -1, - newest_activity_time: naive_now(), - user_id: None, - my_vote: None, - subscribed: None, - read: None, - saved: None, - } + .map(|obox: &BaseBox| { + let page = obox.clone().to_concrete::().unwrap(); + PostView::from_page(&page) }) .collect(); - Ok(GetPostsResponse { posts }) + Ok(GetPostsResponse { posts: posts? }) } pub fn get_remote_community(identifier: &str) -> Result { - let community = - fetch_remote_object::>(&get_remote_community_uri(identifier))?; - let followers_uri = &community.extension.get_followers().unwrap().to_string(); - let outbox_uri = &community.extension.get_outbox().to_string(); - let outbox = fetch_remote_object::(outbox_uri)?; - let followers = fetch_remote_object::(followers_uri)?; + let group = fetch_remote_object::(&get_remote_community_uri(identifier))?; // TODO: this is only for testing until we can call that function from GetPosts // (once string ids are supported) //dbg!(get_remote_community_posts(identifier)?); + let (_, domain) = split_identifier(identifier); Ok(GetCommunityResponse { moderators: vec![], admins: vec![], - community: CommunityView { - // TODO: we need to merge id and name into a single thing (stuff like @user@instance.com) - id: 1337, //community.object_props.get_id() - name: identifier.to_string(), - title: community - .as_ref() - .get_name_xsd_string() - .unwrap() - .to_string(), - description: community - .as_ref() - .get_summary_xsd_string() - .map(|s| s.to_string()), - category_id: -1, - creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri() - removed: false, - published: community - .as_ref() - .get_published() - .unwrap() - .as_ref() - .naive_local() - .to_owned(), - updated: community - .as_ref() - .get_updated() - .map(|u| u.as_ref().to_owned().naive_local()), - deleted: false, - nsfw: false, - creator_name: "".to_string(), - creator_avatar: None, - category_name: "".to_string(), - number_of_subscribers: *followers - .collection_props - .get_total_items() - .unwrap() - .as_ref() as i64, // TODO: need to use the same type - number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64, - number_of_comments: -1, - hot_rank: -1, - user_id: None, - subscribed: None, - }, + community: CommunityView::from_group(&group, &domain)?, online: 0, }) } diff --git a/server/src/routes/federation.rs b/server/src/routes/federation.rs index 06a88e2b..28041d31 100644 --- a/server/src/routes/federation.rs +++ b/server/src/routes/federation.rs @@ -12,9 +12,13 @@ pub fn config(cfg: &mut web::ServiceConfig) { if Settings::get().federation.enabled { println!("federation enabled, host is {}", Settings::get().hostname); cfg + .route( + "/federation/communities", + web::get().to(apub::community::get_apub_community_list), + ) .route( "/federation/c/{community_name}", - web::get().to(apub::community::get_apub_community), + web::get().to(apub::community::get_apub_community_http), ) .route( "/federation/c/{community_name}/followers", diff --git a/server/src/routes/nodeinfo.rs b/server/src/routes/nodeinfo.rs index abfae1ed..679837ee 100644 --- a/server/src/routes/nodeinfo.rs +++ b/server/src/routes/nodeinfo.rs @@ -60,6 +60,13 @@ async fn node_info( local_comments: site_view.number_of_comments, open_registrations: site_view.open_registration, }, + metadata: NodeInfoMetadata { + community_list_url: Some(format!( + "{}://{}/federation/communities", + get_apub_protocol_string(), + Settings::get().hostname + )), + }, }) }) .await @@ -85,6 +92,7 @@ pub struct NodeInfo { pub software: NodeInfoSoftware, pub protocols: Vec, pub usage: NodeInfoUsage, + pub metadata: NodeInfoMetadata, } #[derive(Serialize, Deserialize, Debug)] @@ -106,3 +114,8 @@ pub struct NodeInfoUsage { pub struct NodeInfoUsers { pub total: i64, } + +#[derive(Serialize, Deserialize, Debug)] +pub struct NodeInfoMetadata { + pub community_list_url: Option, +}