From: Felix Ableitner <me@nutomic.com>
Date: Tue, 7 Apr 2020 16:47:19 +0000 (+0200)
Subject: Store remote communities/posts in db, federate posts!
X-Git-Url: http://these/git/ui/assets/static/git-logo.png?a=commitdiff_plain;h=b7103a7e1481e8d7fb2a36938faba520a8cbe013;p=lemmy.git

Store remote communities/posts in db, federate posts!
---

diff --git a/server/src/api/community.rs b/server/src/api/community.rs
index 0ba7effc..3edecb4f 100644
--- a/server/src/api/community.rs
+++ b/server/src/api/community.rs
@@ -1,9 +1,8 @@
 use super::*;
-use crate::apub::puller::{fetch_all_communities, fetch_remote_community};
-use crate::apub::{gen_keypair_str, make_apub_endpoint, EndpointType};
-use crate::settings::Settings;
+use crate::apub::{format_community_name, gen_keypair_str, make_apub_endpoint, EndpointType};
 use diesel::PgConnection;
 use std::str::FromStr;
+use url::Url;
 
 #[derive(Serialize, Deserialize)]
 pub struct GetCommunity {
@@ -41,7 +40,6 @@ pub struct ListCommunities {
   pub page: Option<i64>,
   pub limit: Option<i64>,
   pub auth: Option<String>,
-  pub local_only: Option<bool>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -121,13 +119,6 @@ impl Perform<GetCommunityResponse> for Oper<GetCommunity> {
   fn perform(&self, conn: &PgConnection) -> Result<GetCommunityResponse, Error> {
     let data: &GetCommunity = &self.data;
 
-    if data.name.is_some()
-      && Settings::get().federation.enabled
-      && data.name.as_ref().unwrap().contains('@')
-    {
-      return fetch_remote_community(data.name.as_ref().unwrap());
-    }
-
     let user_id: Option<i32> = match &data.auth {
       Some(auth) => match Claims::decode(&auth) {
         Ok(claims) => {
@@ -139,25 +130,25 @@ impl Perform<GetCommunityResponse> for Oper<GetCommunity> {
       None => None,
     };
 
-    let community_id = match data.id {
-      Some(id) => id,
+    let community = match data.id {
+      Some(id) => Community::read(&conn, id)?,
       None => {
         match Community::read_from_name(
           &conn,
           data.name.to_owned().unwrap_or_else(|| "main".to_string()),
         ) {
-          Ok(community) => community.id,
+          Ok(community) => community,
           Err(_e) => return Err(APIError::err("couldnt_find_community").into()),
         }
       }
     };
 
-    let community_view = match CommunityView::read(&conn, community_id, user_id) {
+    let mut community_view = match CommunityView::read(&conn, community.id, user_id) {
       Ok(community) => community,
       Err(_e) => return Err(APIError::err("couldnt_find_community").into()),
     };
 
-    let moderators = match CommunityModeratorView::for_community(&conn, community_id) {
+    let moderators = match CommunityModeratorView::for_community(&conn, community.id) {
       Ok(moderators) => moderators,
       Err(_e) => return Err(APIError::err("couldnt_find_community").into()),
     };
@@ -168,6 +159,12 @@ impl Perform<GetCommunityResponse> for Oper<GetCommunity> {
     let creator_user = admins.remove(creator_index);
     admins.insert(0, creator_user);
 
+    if !community.local {
+      let domain = Url::parse(&community.actor_id)?;
+      community_view.name =
+        format_community_name(&community_view.name.to_string(), domain.host_str().unwrap());
+    }
+
     // Return the jwt
     Ok(GetCommunityResponse {
       community: community_view,
@@ -226,6 +223,7 @@ impl Perform<CommunityResponse> for Oper<CreateCommunity> {
       private_key: Some(community_private_key),
       public_key: Some(community_public_key),
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = match Community::create(&conn, &community_form) {
@@ -323,6 +321,7 @@ impl Perform<CommunityResponse> for Oper<EditCommunity> {
       private_key: read_community.private_key,
       public_key: read_community.public_key,
       last_refreshed_at: None,
+      published: None,
     };
 
     let _updated_community = match Community::update(&conn, data.edit_id, &community_form) {
@@ -358,13 +357,6 @@ impl Perform<ListCommunitiesResponse> for Oper<ListCommunities> {
   fn perform(&self, conn: &PgConnection) -> Result<ListCommunitiesResponse, Error> {
     let data: &ListCommunities = &self.data;
 
-    let local_only = data.local_only.unwrap_or(false);
-    if Settings::get().federation.enabled && !local_only {
-      return Ok(ListCommunitiesResponse {
-        communities: fetch_all_communities()?,
-      });
-    }
-
     let user_claims: Option<Claims> = match &data.auth {
       Some(auth) => match Claims::decode(&auth) {
         Ok(claims) => Some(claims.claims),
@@ -591,6 +583,7 @@ impl Perform<GetCommunityResponse> for Oper<TransferCommunity> {
       private_key: read_community.private_key,
       public_key: read_community.public_key,
       last_refreshed_at: None,
+      published: None,
     };
 
     let _updated_community = match Community::update(&conn, data.community_id, &community_form) {
diff --git a/server/src/api/post.rs b/server/src/api/post.rs
index cfb71941..18c33dc1 100644
--- a/server/src/api/post.rs
+++ b/server/src/api/post.rs
@@ -1,5 +1,4 @@
 use super::*;
-use crate::settings::Settings;
 use diesel::PgConnection;
 use std::str::FromStr;
 
@@ -133,6 +132,7 @@ impl Perform<PostResponse> for Oper<CreatePost> {
       thumbnail_url: pictshare_thumbnail,
       ap_id: "changeme".into(),
       local: true,
+      published: None,
     };
 
     let inserted_post = match Post::create(&conn, &post_form) {
@@ -228,11 +228,6 @@ impl Perform<GetPostsResponse> for Oper<GetPosts> {
   fn perform(&self, conn: &PgConnection) -> Result<GetPostsResponse, Error> {
     let data: &GetPosts = &self.data;
 
-    if Settings::get().federation.enabled {
-      // TODO: intercept here (but the type is wrong)
-      //get_remote_community_posts(get_posts.community_id.unwrap())
-    }
-
     let user_claims: Option<Claims> = match &data.auth {
       Some(auth) => match Claims::decode(&auth) {
         Ok(claims) => Some(claims.claims),
@@ -398,6 +393,7 @@ impl Perform<PostResponse> for Oper<EditPost> {
       thumbnail_url: pictshare_thumbnail,
       ap_id: read_post.ap_id,
       local: read_post.local,
+      published: None,
     };
 
     let _updated_post = match Post::update(&conn, data.edit_id, &post_form) {
diff --git a/server/src/api/user.rs b/server/src/api/user.rs
index dda1d9ad..fbdead53 100644
--- a/server/src/api/user.rs
+++ b/server/src/api/user.rs
@@ -317,6 +317,7 @@ impl Perform<LoginResponse> for Oper<Register> {
           private_key: Some(community_private_key),
           public_key: Some(community_public_key),
           last_refreshed_at: None,
+          published: None,
         };
         Community::create(&conn, &community_form).unwrap()
       }
diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs
index aa6d832f..eb98a6f6 100644
--- a/server/src/apub/community.rs
+++ b/server/src/apub/community.rs
@@ -1,8 +1,8 @@
 use crate::apub::puller::fetch_remote_object;
 use crate::apub::*;
 use crate::convert_datetime;
-use crate::db::community::Community;
-use crate::db::community_view::{CommunityFollowerView, CommunityView};
+use crate::db::community::{Community, CommunityForm};
+use crate::db::community_view::CommunityFollowerView;
 use crate::db::establish_unpooled_connection;
 use crate::db::post::Post;
 use crate::settings::Settings;
@@ -85,47 +85,34 @@ impl Community {
   }
 }
 
-impl CommunityView {
-  pub fn from_group(group: &GroupExt, domain: &str) -> Result<CommunityView, Error> {
+impl CommunityForm {
+  pub fn from_group(group: &GroupExt) -> Result<Self, Error> {
     let followers_uri = &group.extension.get_followers().unwrap().to_string();
     let outbox_uri = &group.extension.get_outbox().to_string();
-    let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
-    let followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
+    let _outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
+    let _followers = fetch_remote_object::<UnorderedCollection>(followers_uri)?;
     let oprops = &group.base.object_props;
     let aprops = &group.extension;
-    Ok(CommunityView {
-      // TODO: we need to merge id and name into a single thing (stuff like @user@instance.com)
-      id: 1337, //community.object_props.get_id()
-      name: format_community_name(&oprops.get_name_xsd_string().unwrap().to_string(), domain),
+    Ok(CommunityForm {
+      name: oprops.get_name_xsd_string().unwrap().to_string(),
       title: aprops.get_preferred_username().unwrap().to_string(),
       description: oprops.get_summary_xsd_string().map(|s| s.to_string()),
-      category_id: -1,
-      creator_id: -1, //community.object_props.get_attributed_to_xsd_any_uri()
-      removed: false,
+      category_id: 1,
+      creator_id: 2, //community.object_props.get_attributed_to_xsd_any_uri()
+      removed: None,
       published: oprops
         .get_published()
-        .unwrap()
-        .as_ref()
-        .naive_local()
-        .to_owned(),
+        .map(|u| u.as_ref().to_owned().naive_local()),
       updated: oprops
         .get_updated()
         .map(|u| u.as_ref().to_owned().naive_local()),
-      deleted: false,
+      deleted: None,
       nsfw: false,
-      creator_name: "".to_string(),
-      creator_avatar: None,
-      category_name: "".to_string(),
-      number_of_subscribers: *followers
-        .collection_props
-        .get_total_items()
-        .unwrap()
-        .as_ref() as i64,
-      number_of_posts: *outbox.collection_props.get_total_items().unwrap().as_ref() as i64,
-      number_of_comments: -1,
-      hot_rank: -1,
-      user_id: None,
-      subscribed: None,
+      actor_id: oprops.get_id().unwrap().to_string(),
+      local: false,
+      private_key: None,
+      public_key: None,
+      last_refreshed_at: None,
     })
   }
 }
diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs
index 1669ee89..7de8f7b1 100644
--- a/server/src/apub/mod.rs
+++ b/server/src/apub/mod.rs
@@ -99,14 +99,3 @@ pub fn get_following_instances() -> Vec<&'static str> {
     .split(',')
     .collect()
 }
-
-/// Returns a tuple of (username, domain) from an identifier like "main@dev.lemmy.ml"
-fn split_identifier(identifier: &str) -> (String, String) {
-  let x: Vec<&str> = identifier.split('@').collect();
-  (x[0].replace("!", ""), x[1].to_string())
-}
-
-fn get_remote_community_uri(identifier: &str) -> String {
-  let (name, domain) = split_identifier(identifier);
-  format!("http://{}/federation/c/{}", domain, name)
-}
diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs
index 94f3b00e..4c788418 100644
--- a/server/src/apub/post.rs
+++ b/server/src/apub/post.rs
@@ -1,7 +1,6 @@
 use crate::apub::{create_apub_response, make_apub_endpoint, EndpointType};
-use crate::db::post::Post;
-use crate::db::post_view::PostView;
-use crate::{convert_datetime, naive_now};
+use crate::convert_datetime;
+use crate::db::post::{Post, PostForm};
 use activitystreams::{object::properties::ObjectProperties, object::Page};
 use actix_web::body::Body;
 use actix_web::web::Path;
@@ -61,53 +60,32 @@ impl Post {
   }
 }
 
-impl PostView {
-  pub fn from_page(page: &Page) -> Result<PostView, Error> {
+impl PostForm {
+  pub fn from_page(page: &Page) -> Result<PostForm, Error> {
     let oprops = &page.object_props;
-    Ok(PostView {
-      id: -1,
+    Ok(PostForm {
       name: oprops.get_name_xsd_string().unwrap().to_string(),
       url: oprops.get_url_xsd_any_uri().map(|u| u.to_string()),
       body: oprops.get_content_xsd_string().map(|c| c.to_string()),
-      creator_id: -1,
+      creator_id: 2,
       community_id: -1,
-      removed: false,
-      locked: false,
+      removed: None,
+      locked: None,
       published: oprops
         .get_published()
-        .unwrap()
-        .as_ref()
-        .naive_local()
-        .to_owned(),
+        .map(|u| u.as_ref().to_owned().naive_local()),
       updated: oprops
         .get_updated()
         .map(|u| u.as_ref().to_owned().naive_local()),
-      deleted: false,
+      deleted: None,
       nsfw: false,
-      stickied: false,
+      stickied: None,
       embed_title: None,
       embed_description: None,
       embed_html: None,
       thumbnail_url: None,
-      banned: false,
-      banned_from_community: false,
-      creator_name: "".to_string(),
-      creator_avatar: None,
-      community_name: "".to_string(),
-      community_removed: false,
-      community_deleted: false,
-      community_nsfw: false,
-      number_of_comments: -1,
-      score: -1,
-      upvotes: -1,
-      downvotes: -1,
-      hot_rank: -1,
-      newest_activity_time: naive_now(),
-      user_id: None,
-      my_vote: None,
-      subscribed: None,
-      read: None,
-      saved: None,
+      ap_id: oprops.get_id().unwrap().to_string(),
+      local: false,
     })
   }
 }
diff --git a/server/src/apub/puller.rs b/server/src/apub/puller.rs
index fde914bc..9796faf4 100644
--- a/server/src/apub/puller.rs
+++ b/server/src/apub/puller.rs
@@ -1,16 +1,15 @@
-use crate::api::community::GetCommunityResponse;
-use crate::api::post::GetPostsResponse;
 use crate::apub::*;
-use crate::db::community_view::CommunityView;
-use crate::db::post_view::PostView;
+use crate::db::community::{Community, CommunityForm};
+use crate::db::post::{Post, PostForm};
+use crate::db::Crud;
 use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown};
 use crate::settings::Settings;
 use activitystreams::collection::{OrderedCollection, UnorderedCollection};
 use activitystreams::object::Page;
 use activitystreams::BaseBox;
+use diesel::PgConnection;
 use failure::Error;
 use isahc::prelude::*;
-use log::warn;
 use serde::Deserialize;
 use std::time::Duration;
 
@@ -24,7 +23,7 @@ fn fetch_node_info(domain: &str) -> Result<NodeInfo, Error> {
   Ok(fetch_remote_object::<NodeInfo>(&well_known.links.href)?)
 }
 
-fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, Error> {
+fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityForm>, Error> {
   let node_info = fetch_node_info(domain)?;
 
   if let Some(community_list_url) = node_info.metadata.community_list_url {
@@ -33,10 +32,10 @@ fn fetch_communities_from_instance(domain: &str) -> Result<Vec<CommunityView>, E
       .collection_props
       .get_many_items_base_boxes()
       .unwrap();
-    let communities: Result<Vec<CommunityView>, Error> = object_boxes
-      .map(|c| -> Result<CommunityView, Error> {
+    let communities: Result<Vec<CommunityForm>, Error> = object_boxes
+      .map(|c| {
         let group = c.to_owned().to_concrete::<GroupExt>()?;
-        CommunityView::from_group(&group, domain)
+        CommunityForm::from_group(&group)
       })
       .collect();
     Ok(communities?)
@@ -69,44 +68,51 @@ where
   Ok(res)
 }
 
-pub fn fetch_remote_community_posts(identifier: &str) -> Result<GetPostsResponse, Error> {
-  let community = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
+fn fetch_remote_community_posts(instance: &str, community: &str) -> Result<Vec<PostForm>, Error> {
+  let endpoint = format!("http://{}/federation/c/{}", instance, community);
+  let community = fetch_remote_object::<GroupExt>(&endpoint)?;
   let outbox_uri = &community.extension.get_outbox().to_string();
   let outbox = fetch_remote_object::<OrderedCollection>(outbox_uri)?;
   let items = outbox.collection_props.get_many_items_base_boxes();
 
-  let posts: Result<Vec<PostView>, Error> = items
+  let posts = items
     .unwrap()
     .map(|obox: &BaseBox| {
       let page = obox.clone().to_concrete::<Page>().unwrap();
-      PostView::from_page(&page)
+      PostForm::from_page(&page)
     })
-    .collect();
-  Ok(GetPostsResponse { posts: posts? })
+    .collect::<Result<Vec<PostForm>, Error>>()?;
+  Ok(posts)
 }
 
-pub fn fetch_remote_community(identifier: &str) -> Result<GetCommunityResponse, failure::Error> {
-  let group = fetch_remote_object::<GroupExt>(&get_remote_community_uri(identifier))?;
-  // TODO: this is only for testing until we can call that function from GetPosts
-  // (once string ids are supported)
-  //dbg!(get_remote_community_posts(identifier)?);
-
-  let (_, domain) = split_identifier(identifier);
-  Ok(GetCommunityResponse {
-    moderators: vec![],
-    admins: vec![],
-    community: CommunityView::from_group(&group, &domain)?,
-    online: 0,
-  })
-}
-
-pub fn fetch_all_communities() -> Result<Vec<CommunityView>, Error> {
-  let mut communities_list: Vec<CommunityView> = vec![];
+// TODO: in the future, this should only be done when an instance is followed for the first time
+//       after that, we should rely in the inbox, and fetch on demand when needed
+pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> {
   for instance in &get_following_instances() {
-    match fetch_communities_from_instance(instance) {
-      Ok(mut c) => communities_list.append(c.as_mut()),
-      Err(e) => warn!("Failed to fetch instance list from remote instance: {}", e),
-    };
+    let communities = fetch_communities_from_instance(instance)?;
+
+    for community in &communities {
+      let existing = Community::read_from_actor_id(conn, &community.actor_id);
+      let community_id = match existing {
+        // TODO: should make sure that this is actually a `NotFound` error
+        Err(_) => Community::create(conn, community)?.id,
+        Ok(c) => Community::update(conn, c.id, community)?.id,
+      };
+      let mut posts = fetch_remote_community_posts(instance, &community.name)?;
+      for post_ in &mut posts {
+        post_.community_id = community_id;
+        let existing = Post::read_from_apub_id(conn, &post_.ap_id);
+        match existing {
+          // TODO: should make sure that this is actually a `NotFound` error
+          Err(_) => {
+            Post::create(conn, post_)?;
+          }
+          Ok(p) => {
+            Post::update(conn, p.id, post_)?;
+          }
+        }
+      }
+    }
   }
-  Ok(communities_list)
+  Ok(())
 }
diff --git a/server/src/db/code_migrations.rs b/server/src/db/code_migrations.rs
index 3575462e..a13a9964 100644
--- a/server/src/db/code_migrations.rs
+++ b/server/src/db/code_migrations.rs
@@ -93,6 +93,7 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> {
       private_key: Some(community_private_key),
       public_key: Some(community_public_key),
       last_refreshed_at: Some(naive_now()),
+      published: None,
     };
 
     Community::update(&conn, ccommunity.id, &form)?;
diff --git a/server/src/db/comment.rs b/server/src/db/comment.rs
index 4925aabc..0b8a2e20 100644
--- a/server/src/db/comment.rs
+++ b/server/src/db/comment.rs
@@ -247,6 +247,7 @@ mod tests {
       private_key: None,
       public_key: None,
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = Community::create(&conn, &new_community).unwrap();
@@ -269,6 +270,7 @@ mod tests {
       thumbnail_url: None,
       ap_id: "changeme".into(),
       local: true,
+      published: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/comment_view.rs b/server/src/db/comment_view.rs
index bbeeecde..11507dcd 100644
--- a/server/src/db/comment_view.rs
+++ b/server/src/db/comment_view.rs
@@ -474,6 +474,7 @@ mod tests {
       private_key: None,
       public_key: None,
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = Community::create(&conn, &new_community).unwrap();
@@ -496,6 +497,7 @@ mod tests {
       thumbnail_url: None,
       ap_id: "changeme".into(),
       local: true,
+      published: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/community.rs b/server/src/db/community.rs
index 15915715..1b81ac8f 100644
--- a/server/src/db/community.rs
+++ b/server/src/db/community.rs
@@ -22,7 +22,7 @@ pub struct Community {
   pub last_refreshed_at: chrono::NaiveDateTime,
 }
 
-#[derive(Insertable, AsChangeset, Clone, Serialize, Deserialize)]
+#[derive(Insertable, AsChangeset, Clone, Serialize, Deserialize, Debug)]
 #[table_name = "community"]
 pub struct CommunityForm {
   pub name: String,
@@ -31,6 +31,7 @@ pub struct CommunityForm {
   pub category_id: i32,
   pub creator_id: i32,
   pub removed: Option<bool>,
+  pub published: Option<chrono::NaiveDateTime>,
   pub updated: Option<chrono::NaiveDateTime>,
   pub deleted: Option<bool>,
   pub nsfw: bool,
@@ -79,6 +80,13 @@ impl Community {
       .first::<Self>(conn)
   }
 
+  pub fn read_from_actor_id(conn: &PgConnection, community_id: &str) -> Result<Self, Error> {
+    use crate::schema::community::dsl::*;
+    community
+      .filter(actor_id.eq(community_id))
+      .first::<Self>(conn)
+  }
+
   pub fn list(conn: &PgConnection) -> Result<Vec<Self>, Error> {
     use crate::schema::community::dsl::*;
     community.load::<Community>(conn)
@@ -271,6 +279,7 @@ mod tests {
       private_key: None,
       public_key: None,
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = Community::create(&conn, &new_community).unwrap();
diff --git a/server/src/db/moderator.rs b/server/src/db/moderator.rs
index 88262112..d56e3914 100644
--- a/server/src/db/moderator.rs
+++ b/server/src/db/moderator.rs
@@ -505,6 +505,7 @@ mod tests {
       private_key: None,
       public_key: None,
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = Community::create(&conn, &new_community).unwrap();
@@ -527,6 +528,7 @@ mod tests {
       thumbnail_url: None,
       ap_id: "changeme".into(),
       local: true,
+      published: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/post.rs b/server/src/db/post.rs
index 469fa819..bf455b1a 100644
--- a/server/src/db/post.rs
+++ b/server/src/db/post.rs
@@ -27,7 +27,7 @@ pub struct Post {
   pub local: bool,
 }
 
-#[derive(Insertable, AsChangeset, Clone)]
+#[derive(Insertable, AsChangeset, Clone, Debug)]
 #[table_name = "post"]
 pub struct PostForm {
   pub name: String,
@@ -37,6 +37,7 @@ pub struct PostForm {
   pub community_id: i32,
   pub removed: Option<bool>,
   pub locked: Option<bool>,
+  pub published: Option<chrono::NaiveDateTime>,
   pub updated: Option<chrono::NaiveDateTime>,
   pub deleted: Option<bool>,
   pub nsfw: bool,
@@ -65,6 +66,11 @@ impl Post {
       .load::<Self>(conn)
   }
 
+  pub fn read_from_apub_id(conn: &PgConnection, object_id: &str) -> Result<Self, Error> {
+    use crate::schema::post::dsl::*;
+    post.filter(ap_id.eq(object_id)).first::<Self>(conn)
+  }
+
   pub fn update_ap_id(conn: &PgConnection, post_id: i32) -> Result<Self, Error> {
     use crate::schema::post::dsl::*;
 
@@ -105,11 +111,13 @@ impl Crud<PostForm> for Post {
 
   fn create(conn: &PgConnection, new_post: &PostForm) -> Result<Self, Error> {
     use crate::schema::post::dsl::*;
+    dbg!(&new_post);
     insert_into(post).values(new_post).get_result::<Self>(conn)
   }
 
   fn update(conn: &PgConnection, post_id: i32, new_post: &PostForm) -> Result<Self, Error> {
     use crate::schema::post::dsl::*;
+    dbg!(&new_post);
     diesel::update(post.find(post_id))
       .set(new_post)
       .get_result::<Self>(conn)
@@ -280,6 +288,7 @@ mod tests {
       private_key: None,
       public_key: None,
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = Community::create(&conn, &new_community).unwrap();
@@ -302,6 +311,7 @@ mod tests {
       thumbnail_url: None,
       ap_id: "changeme".into(),
       local: true,
+      published: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/post_view.rs b/server/src/db/post_view.rs
index 420e46df..3c155b00 100644
--- a/server/src/db/post_view.rs
+++ b/server/src/db/post_view.rs
@@ -399,6 +399,7 @@ mod tests {
       private_key: None,
       public_key: None,
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = Community::create(&conn, &new_community).unwrap();
@@ -421,6 +422,7 @@ mod tests {
       thumbnail_url: None,
       ap_id: "changeme".into(),
       local: true,
+      published: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/user_mention.rs b/server/src/db/user_mention.rs
index 20eed23e..aea1e228 100644
--- a/server/src/db/user_mention.rs
+++ b/server/src/db/user_mention.rs
@@ -131,6 +131,7 @@ mod tests {
       private_key: None,
       public_key: None,
       last_refreshed_at: None,
+      published: None,
     };
 
     let inserted_community = Community::create(&conn, &new_community).unwrap();
@@ -153,6 +154,7 @@ mod tests {
       thumbnail_url: None,
       ap_id: "changeme".into(),
       local: true,
+      published: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/main.rs b/server/src/main.rs
index f2a19d30..c1bec7e0 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -6,16 +6,21 @@ use actix::prelude::*;
 use actix_web::*;
 use diesel::r2d2::{ConnectionManager, Pool};
 use diesel::PgConnection;
+use failure::Error;
+use lemmy_server::apub::puller::fetch_all;
 use lemmy_server::db::code_migrations::run_advanced_migrations;
 use lemmy_server::routes::{api, federation, feeds, index, nodeinfo, webfinger, websocket};
 use lemmy_server::settings::Settings;
 use lemmy_server::websocket::server::*;
-use std::io;
+use log::warn;
+use std::thread;
+use std::thread::sleep;
+use std::time::Duration;
 
 embed_migrations!();
 
 #[actix_rt::main]
-async fn main() -> io::Result<()> {
+async fn main() -> Result<(), Error> {
   env_logger::init();
   let settings = Settings::get();
 
@@ -34,36 +39,50 @@ async fn main() -> io::Result<()> {
   // Set up websocket server
   let server = ChatServer::startup(pool.clone()).start();
 
+  // TODO: its probably failing because the other instance is not up yet
+  //       need to make a new thread and wait a bit before fetching
+  thread::spawn(move || {
+    // some work here
+    sleep(Duration::from_secs(5));
+    println!("Fetching apub data");
+    match fetch_all(&conn) {
+      Ok(_) => {}
+      Err(e) => warn!("Error during apub fetch: {}", e),
+    }
+  });
+
   println!(
     "Starting http server at {}:{}",
     settings.bind, settings.port
   );
 
   // Create Http server with websocket support
-  HttpServer::new(move || {
-    App::new()
-      .wrap(middleware::Logger::default())
-      .data(pool.clone())
-      .data(server.clone())
-      // The routes
-      .configure(api::config)
-      .configure(federation::config)
-      .configure(feeds::config)
-      .configure(index::config)
-      .configure(nodeinfo::config)
-      .configure(webfinger::config)
-      .configure(websocket::config)
-      // static files
-      .service(actix_files::Files::new(
-        "/static",
-        settings.front_end_dir.to_owned(),
-      ))
-      .service(actix_files::Files::new(
-        "/docs",
-        settings.front_end_dir.to_owned() + "/documentation",
-      ))
-  })
-  .bind((settings.bind, settings.port))?
-  .run()
-  .await
+  Ok(
+    HttpServer::new(move || {
+      App::new()
+        .wrap(middleware::Logger::default())
+        .data(pool.clone())
+        .data(server.clone())
+        // The routes
+        .configure(api::config)
+        .configure(federation::config)
+        .configure(feeds::config)
+        .configure(index::config)
+        .configure(nodeinfo::config)
+        .configure(webfinger::config)
+        .configure(websocket::config)
+        // static files
+        .service(actix_files::Files::new(
+          "/static",
+          settings.front_end_dir.to_owned(),
+        ))
+        .service(actix_files::Files::new(
+          "/docs",
+          settings.front_end_dir.to_owned() + "/documentation",
+        ))
+    })
+    .bind((settings.bind, settings.port))?
+    .run()
+    .await?,
+  )
 }