1 // This is for db migrations that require code
2 use activitypub_federation::http_signatures::generate_actor_keypair;
4 sql_types::{Nullable, Text},
10 use diesel_async::RunQueryDsl;
11 use lemmy_api_common::{
12 lemmy_db_views::structs::SiteView,
14 generate_followers_url,
16 generate_local_apub_endpoint,
17 generate_shared_inbox_url,
18 generate_site_inbox_url,
22 use lemmy_db_schema::{
24 comment::{Comment, CommentUpdateForm},
25 community::{Community, CommunityUpdateForm},
27 local_site::{LocalSite, LocalSiteInsertForm},
28 local_site_rate_limit::{LocalSiteRateLimit, LocalSiteRateLimitInsertForm},
29 local_user::{LocalUser, LocalUserInsertForm},
30 person::{Person, PersonInsertForm, PersonUpdateForm},
31 post::{Post, PostUpdateForm},
32 private_message::{PrivateMessage, PrivateMessageUpdateForm},
33 site::{Site, SiteInsertForm, SiteUpdateForm},
36 utils::{get_conn, naive_now, DbPool},
38 use lemmy_utils::{error::LemmyError, settings::structs::Settings};
42 pub async fn run_advanced_migrations(
43 pool: &mut DbPool<'_>,
45 ) -> Result<(), LemmyError> {
46 let protocol_and_hostname = &settings.get_protocol_and_hostname();
47 user_updates_2020_04_02(pool, protocol_and_hostname).await?;
48 community_updates_2020_04_02(pool, protocol_and_hostname).await?;
49 post_updates_2020_04_03(pool, protocol_and_hostname).await?;
50 comment_updates_2020_04_03(pool, protocol_and_hostname).await?;
51 private_message_updates_2020_05_05(pool, protocol_and_hostname).await?;
52 post_thumbnail_url_updates_2020_07_27(pool, protocol_and_hostname).await?;
53 apub_columns_2021_02_02(pool).await?;
54 instance_actor_2022_01_28(pool, protocol_and_hostname).await?;
55 regenerate_public_keys_2022_07_05(pool).await?;
56 initialize_local_site_2022_10_10(pool, settings).await?;
61 async fn user_updates_2020_04_02(
62 pool: &mut DbPool<'_>,
63 protocol_and_hostname: &str,
64 ) -> Result<(), LemmyError> {
65 use lemmy_db_schema::schema::person::dsl::{actor_id, local, person};
66 let conn = &mut get_conn(pool).await?;
68 info!("Running user_updates_2020_04_02");
70 // Update the actor_id, private_key, and public_key, last_refreshed_at
71 let incorrect_persons = person
72 .filter(actor_id.like("http://changeme%"))
73 .filter(local.eq(true))
77 for cperson in &incorrect_persons {
78 let keypair = generate_actor_keypair()?;
80 let form = PersonUpdateForm::builder()
81 .actor_id(Some(generate_local_apub_endpoint(
84 protocol_and_hostname,
86 .private_key(Some(Some(keypair.private_key)))
87 .public_key(Some(keypair.public_key))
88 .last_refreshed_at(Some(naive_now()))
91 Person::update(pool, cperson.id, &form).await?;
94 info!("{} person rows updated.", incorrect_persons.len());
99 async fn community_updates_2020_04_02(
100 pool: &mut DbPool<'_>,
101 protocol_and_hostname: &str,
102 ) -> Result<(), LemmyError> {
103 use lemmy_db_schema::schema::community::dsl::{actor_id, community, local};
104 let conn = &mut get_conn(pool).await?;
106 info!("Running community_updates_2020_04_02");
108 // Update the actor_id, private_key, and public_key, last_refreshed_at
109 let incorrect_communities = community
110 .filter(actor_id.like("http://changeme%"))
111 .filter(local.eq(true))
112 .load::<Community>(conn)
115 for ccommunity in &incorrect_communities {
116 let keypair = generate_actor_keypair()?;
117 let community_actor_id = generate_local_apub_endpoint(
118 EndpointType::Community,
120 protocol_and_hostname,
123 let form = CommunityUpdateForm::builder()
124 .actor_id(Some(community_actor_id.clone()))
125 .private_key(Some(Some(keypair.private_key)))
126 .public_key(Some(keypair.public_key))
127 .last_refreshed_at(Some(naive_now()))
130 Community::update(pool, ccommunity.id, &form).await?;
133 info!("{} community rows updated.", incorrect_communities.len());
138 async fn post_updates_2020_04_03(
139 pool: &mut DbPool<'_>,
140 protocol_and_hostname: &str,
141 ) -> Result<(), LemmyError> {
142 use lemmy_db_schema::schema::post::dsl::{ap_id, local, post};
143 let conn = &mut get_conn(pool).await?;
145 info!("Running post_updates_2020_04_03");
148 let incorrect_posts = post
149 .filter(ap_id.like("http://changeme%"))
150 .filter(local.eq(true))
154 for cpost in &incorrect_posts {
155 let apub_id = generate_local_apub_endpoint(
157 &cpost.id.to_string(),
158 protocol_and_hostname,
163 &PostUpdateForm::builder().ap_id(Some(apub_id)).build(),
168 info!("{} post rows updated.", incorrect_posts.len());
173 async fn comment_updates_2020_04_03(
174 pool: &mut DbPool<'_>,
175 protocol_and_hostname: &str,
176 ) -> Result<(), LemmyError> {
177 use lemmy_db_schema::schema::comment::dsl::{ap_id, comment, local};
178 let conn = &mut get_conn(pool).await?;
180 info!("Running comment_updates_2020_04_03");
183 let incorrect_comments = comment
184 .filter(ap_id.like("http://changeme%"))
185 .filter(local.eq(true))
186 .load::<Comment>(conn)
189 for ccomment in &incorrect_comments {
190 let apub_id = generate_local_apub_endpoint(
191 EndpointType::Comment,
192 &ccomment.id.to_string(),
193 protocol_and_hostname,
198 &CommentUpdateForm::builder().ap_id(Some(apub_id)).build(),
203 info!("{} comment rows updated.", incorrect_comments.len());
208 async fn private_message_updates_2020_05_05(
209 pool: &mut DbPool<'_>,
210 protocol_and_hostname: &str,
211 ) -> Result<(), LemmyError> {
212 use lemmy_db_schema::schema::private_message::dsl::{ap_id, local, private_message};
213 let conn = &mut get_conn(pool).await?;
215 info!("Running private_message_updates_2020_05_05");
218 let incorrect_pms = private_message
219 .filter(ap_id.like("http://changeme%"))
220 .filter(local.eq(true))
221 .load::<PrivateMessage>(conn)
224 for cpm in &incorrect_pms {
225 let apub_id = generate_local_apub_endpoint(
226 EndpointType::PrivateMessage,
228 protocol_and_hostname,
230 PrivateMessage::update(
233 &PrivateMessageUpdateForm::builder()
234 .ap_id(Some(apub_id))
240 info!("{} private message rows updated.", incorrect_pms.len());
245 async fn post_thumbnail_url_updates_2020_07_27(
246 pool: &mut DbPool<'_>,
247 protocol_and_hostname: &str,
248 ) -> Result<(), LemmyError> {
249 use lemmy_db_schema::schema::post::dsl::{post, thumbnail_url};
250 let conn = &mut get_conn(pool).await?;
252 info!("Running post_thumbnail_url_updates_2020_07_27");
254 let domain_prefix = format!("{protocol_and_hostname}/pictrs/image/",);
256 let incorrect_thumbnails = post.filter(thumbnail_url.not_like("http%"));
258 // Prepend the rows with the update
259 let res = diesel::update(incorrect_thumbnails)
263 .into_sql::<Nullable<Text>>()
264 .concat(thumbnail_url),
267 .get_results::<Post>(conn)
270 info!("{} Post thumbnail_url rows updated.", res.len());
275 /// We are setting inbox and follower URLs for local and remote actors alike, because for now
276 /// all federated instances are also Lemmy and use the same URL scheme.
277 async fn apub_columns_2021_02_02(pool: &mut DbPool<'_>) -> Result<(), LemmyError> {
278 let conn = &mut get_conn(pool).await?;
279 info!("Running apub_columns_2021_02_02");
281 use lemmy_db_schema::schema::person::dsl::{inbox_url, person, shared_inbox_url};
283 .filter(inbox_url.like("http://changeme%"))
284 .load::<Person>(conn)
288 let inbox_url_ = generate_inbox_url(&p.actor_id)?;
289 let shared_inbox_url_ = generate_shared_inbox_url(&p.actor_id)?;
290 diesel::update(person.find(p.id))
292 inbox_url.eq(inbox_url_),
293 shared_inbox_url.eq(shared_inbox_url_),
295 .get_result::<Person>(conn)
301 use lemmy_db_schema::schema::community::dsl::{
307 let communities = community
308 .filter(inbox_url.like("http://changeme%"))
309 .load::<Community>(conn)
312 for c in &communities {
313 let followers_url_ = generate_followers_url(&c.actor_id)?;
314 let inbox_url_ = generate_inbox_url(&c.actor_id)?;
315 let shared_inbox_url_ = generate_shared_inbox_url(&c.actor_id)?;
316 diesel::update(community.find(c.id))
318 followers_url.eq(followers_url_),
319 inbox_url.eq(inbox_url_),
320 shared_inbox_url.eq(shared_inbox_url_),
322 .get_result::<Community>(conn)
330 /// Site object turns into an actor, so that things like instance description can be federated. This
331 /// means we need to add actor columns to the site table, and initialize them with correct values.
332 /// Before this point, there is only a single value in the site table which refers to the local
333 /// Lemmy instance, so thats all we need to update.
334 async fn instance_actor_2022_01_28(
335 pool: &mut DbPool<'_>,
336 protocol_and_hostname: &str,
337 ) -> Result<(), LemmyError> {
338 info!("Running instance_actor_2021_09_29");
339 if let Ok(site_view) = SiteView::read_local(pool).await {
340 let site = site_view.site;
341 // if site already has public key, we dont need to do anything here
342 if !site.public_key.is_empty() {
345 let key_pair = generate_actor_keypair()?;
346 let actor_id = Url::parse(protocol_and_hostname)?;
347 let site_form = SiteUpdateForm::builder()
348 .actor_id(Some(actor_id.clone().into()))
349 .last_refreshed_at(Some(naive_now()))
350 .inbox_url(Some(generate_site_inbox_url(&actor_id.into())?))
351 .private_key(Some(Some(key_pair.private_key)))
352 .public_key(Some(key_pair.public_key))
354 Site::update(pool, site.id, &site_form).await?;
359 /// Fix for bug #2347, which can result in community/person public keys being overwritten with
360 /// empty string when the database value is updated. We go through all actors, and if the public
361 /// key field is empty, generate a new keypair. It would be possible to regenerate only the pubkey,
362 /// but thats more complicated and has no benefit, as federation is already broken for these actors.
363 /// https://github.com/LemmyNet/lemmy/issues/2347
364 async fn regenerate_public_keys_2022_07_05(pool: &mut DbPool<'_>) -> Result<(), LemmyError> {
365 let conn = &mut get_conn(pool).await?;
366 info!("Running regenerate_public_keys_2022_07_05");
369 // update communities with empty pubkey
370 use lemmy_db_schema::schema::community::dsl::{community, local, public_key};
371 let communities: Vec<Community> = community
372 .filter(local.eq(true))
373 .filter(public_key.eq(""))
374 .load::<Community>(conn)
376 for community_ in communities {
378 "local community {} has empty public key field, regenerating key",
381 let key_pair = generate_actor_keypair()?;
382 let form = CommunityUpdateForm::builder()
383 .public_key(Some(key_pair.public_key))
384 .private_key(Some(Some(key_pair.private_key)))
386 Community::update(&mut conn.into(), community_.id, &form).await?;
391 // update persons with empty pubkey
392 use lemmy_db_schema::schema::person::dsl::{local, person, public_key};
394 .filter(local.eq(true))
395 .filter(public_key.eq(""))
396 .load::<Person>(conn)
398 for person_ in persons {
400 "local user {} has empty public key field, regenerating key",
403 let key_pair = generate_actor_keypair()?;
404 let form = PersonUpdateForm::builder()
405 .public_key(Some(key_pair.public_key))
406 .private_key(Some(Some(key_pair.private_key)))
408 Person::update(pool, person_.id, &form).await?;
414 /// This ensures that your local site is initialized and exists.
416 /// If a site already exists, the DB migration should generate a local_site row.
417 /// This will only be run for brand new sites.
418 async fn initialize_local_site_2022_10_10(
419 pool: &mut DbPool<'_>,
421 ) -> Result<(), LemmyError> {
422 info!("Running initialize_local_site_2022_10_10");
424 // Check to see if local_site exists
425 if LocalSite::read(pool).await.is_ok() {
428 info!("No Local Site found, creating it.");
430 let domain = settings
431 .get_hostname_without_port()
432 .expect("must have domain");
434 // Upsert this to the instance table
435 let instance = Instance::read_or_create(pool, domain).await?;
437 if let Some(setup) = &settings.setup {
438 let person_keypair = generate_actor_keypair()?;
439 let person_actor_id = generate_local_apub_endpoint(
440 EndpointType::Person,
441 &setup.admin_username,
442 &settings.get_protocol_and_hostname(),
445 // Register the user if there's a site setup
446 let person_form = PersonInsertForm::builder()
447 .name(setup.admin_username.clone())
449 .instance_id(instance.id)
450 .actor_id(Some(person_actor_id.clone()))
451 .private_key(Some(person_keypair.private_key))
452 .public_key(person_keypair.public_key)
453 .inbox_url(Some(generate_inbox_url(&person_actor_id)?))
454 .shared_inbox_url(Some(generate_shared_inbox_url(&person_actor_id)?))
456 let person_inserted = Person::create(pool, &person_form).await?;
458 let local_user_form = LocalUserInsertForm::builder()
459 .person_id(person_inserted.id)
460 .password_encrypted(setup.admin_password.clone())
461 .email(setup.admin_email.clone())
463 LocalUser::create(pool, &local_user_form).await?;
466 // Add an entry for the site table
467 let site_key_pair = generate_actor_keypair()?;
468 let site_actor_id = Url::parse(&settings.get_protocol_and_hostname())?;
470 let site_form = SiteInsertForm::builder()
475 .map(|s| s.site_name)
476 .unwrap_or_else(|| "New Site".to_string()),
478 .instance_id(instance.id)
479 .actor_id(Some(site_actor_id.clone().into()))
480 .last_refreshed_at(Some(naive_now()))
481 .inbox_url(Some(generate_site_inbox_url(&site_actor_id.into())?))
482 .private_key(Some(site_key_pair.private_key))
483 .public_key(Some(site_key_pair.public_key))
485 let site = Site::create(pool, &site_form).await?;
487 // Finally create the local_site row
488 let local_site_form = LocalSiteInsertForm::builder()
490 .site_setup(Some(settings.setup.is_some()))
492 let local_site = LocalSite::create(pool, &local_site_form).await?;
494 // Create the rate limit table
495 let local_site_rate_limit_form = LocalSiteRateLimitInsertForm::builder()
496 // TODO these have to be set, because the database defaults are too low for the federation
497 // tests to pass, and there's no way to live update the rate limits without restarting the
499 // This can be removed once live rate limits are enabled.
506 .local_site_id(local_site.id)
508 LocalSiteRateLimit::create(pool, &local_site_rate_limit_form).await?;