From 3735c6fabfd85d6c98c2fc126b70e593db532eb8 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Sat, 18 Feb 2023 09:36:12 -0500 Subject: [PATCH] Adding instance software and version. Fixes #2222 (#2733) * Adding instance software and version. Fixes #2222 * Fix clippy. * Fix clippy 2 * Fixing some more issues. --- Cargo.toml | 2 +- crates/api_common/src/site.rs | 13 ++- crates/apub/src/lib.rs | 13 +-- .../src/impls/federation_allowlist.rs | 19 ++--- crates/db_schema/src/impls/instance.rs | 26 +++--- crates/db_schema/src/schema.rs | 2 + crates/db_schema/src/source/instance.rs | 11 ++- crates/routes/src/nodeinfo.rs | 83 ++++++++++--------- .../down.sql | 2 + .../up.sql | 4 + scripts/fix-clippy.sh | 22 +++-- src/lib.rs | 15 ++-- src/scheduled_tasks.rs | 81 +++++++++++++++++- 13 files changed, 201 insertions(+), 92 deletions(-) create mode 100644 migrations/2023-02-13-221303_add_instance_software_and_version/down.sql create mode 100644 migrations/2023-02-13-221303_add_instance_software_and_version/up.sql diff --git a/Cargo.toml b/Cargo.toml index 123991c7..6fcccc4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,7 @@ tracing-error = "0.2.0" tracing-log = "0.1.3" tracing-subscriber = { version = "0.3.15", features = ["env-filter"] } url = { version = "2.3.1", features = ["serde"] } -reqwest = { version = "0.11.12", features = ["json"] } +reqwest = { version = "0.11.12", features = ["json", "blocking"] } reqwest-middleware = "0.2.0" reqwest-tracing = "0.4.0" clokwerk = "0.3.5" diff --git a/crates/api_common/src/site.rs b/crates/api_common/src/site.rs index 2c96942c..6c967068 100644 --- a/crates/api_common/src/site.rs +++ b/crates/api_common/src/site.rs @@ -1,7 +1,12 @@ use crate::sensitive::Sensitive; use lemmy_db_schema::{ newtypes::{CommentId, CommunityId, LanguageId, PersonId, PostId}, - source::{language::Language, local_site::RegistrationMode, tagline::Tagline}, + source::{ + instance::Instance, + language::Language, + local_site::RegistrationMode, + tagline::Tagline, + }, ListingType, ModlogActionType, SearchType, @@ -239,9 +244,9 @@ pub struct LeaveAdmin { #[derive(Debug, Serialize, Deserialize, Clone)] pub struct FederatedInstances { - pub linked: Vec, - pub allowed: Option>, - pub blocked: Option>, + pub linked: Vec, + pub allowed: Option>, + pub blocked: Option>, } #[derive(Debug, Serialize, Deserialize, Clone)] diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index 53fe7cd3..d277a3bc 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -114,13 +114,13 @@ fn check_apub_id_valid( } if let Some(blocked) = local_site_data.blocked_instances.as_ref() { - if blocked.contains(&domain) { + if blocked.iter().any(|i| domain.eq(&i.domain)) { return Err("Domain is blocked"); } } if let Some(allowed) = local_site_data.allowed_instances.as_ref() { - if !allowed.contains(&domain) { + if !allowed.iter().any(|i| domain.eq(&i.domain)) { return Err("Domain is not in allowlist"); } } @@ -131,8 +131,8 @@ fn check_apub_id_valid( #[derive(Clone)] pub(crate) struct LocalSiteData { local_site: Option, - allowed_instances: Option>, - blocked_instances: Option>, + allowed_instances: Option>, + blocked_instances: Option>, } pub(crate) async fn fetch_local_site_data( @@ -175,7 +175,10 @@ pub(crate) fn check_apub_id_valid_with_strictness( if is_strict { // need to allow this explicitly because apub receive might contain objects from our local // instance. - let mut allowed_and_local = allowed.clone(); + let mut allowed_and_local = allowed + .iter() + .map(|i| i.domain.clone()) + .collect::>(); allowed_and_local.push(local_instance); if !allowed_and_local.contains(&domain) { diff --git a/crates/db_schema/src/impls/federation_allowlist.rs b/crates/db_schema/src/impls/federation_allowlist.rs index c0b4020e..79efecc9 100644 --- a/crates/db_schema/src/impls/federation_allowlist.rs +++ b/crates/db_schema/src/impls/federation_allowlist.rs @@ -59,25 +59,24 @@ mod tests { #[serial] async fn test_allowlist_insert_and_clear() { let pool = &build_db_pool_for_tests().await; - let allowed = Some(vec![ + let domains = vec![ "tld1.xyz".to_string(), "tld2.xyz".to_string(), "tld3.xyz".to_string(), - ]); + ]; + + let allowed = Some(domains.clone()); FederationAllowList::replace(pool, allowed).await.unwrap(); let allows = Instance::allowlist(pool).await.unwrap(); + let allows_domains = allows + .iter() + .map(|i| i.domain.clone()) + .collect::>(); assert_eq!(3, allows.len()); - assert_eq!( - vec![ - "tld1.xyz".to_string(), - "tld2.xyz".to_string(), - "tld3.xyz".to_string() - ], - allows - ); + assert_eq!(domains, allows_domains); // Now test clearing them via Some(empty vec) let clear_allows = Some(Vec::new()); diff --git a/crates/db_schema/src/impls/instance.rs b/crates/db_schema/src/impls/instance.rs index 32d03e38..473ca007 100644 --- a/crates/db_schema/src/impls/instance.rs +++ b/crates/db_schema/src/impls/instance.rs @@ -31,10 +31,10 @@ impl Instance { Self::create(pool, domain).await } pub async fn create_conn(conn: &mut AsyncPgConnection, domain: &str) -> Result { - let form = InstanceForm { - domain: domain.to_string(), - updated: Some(naive_now()), - }; + let form = InstanceForm::builder() + .domain(domain.to_string()) + .updated(Some(naive_now())) + .build(); Self::create_from_form_conn(conn, &form).await } pub async fn delete(pool: &DbPool, instance_id: InstanceId) -> Result { @@ -47,31 +47,31 @@ impl Instance { let conn = &mut get_conn(pool).await?; diesel::delete(instance::table).execute(conn).await } - pub async fn allowlist(pool: &DbPool) -> Result, Error> { + pub async fn allowlist(pool: &DbPool) -> Result, Error> { let conn = &mut get_conn(pool).await?; instance::table .inner_join(federation_allowlist::table) - .select(instance::domain) - .load::(conn) + .select(instance::all_columns) + .get_results(conn) .await } - pub async fn blocklist(pool: &DbPool) -> Result, Error> { + pub async fn blocklist(pool: &DbPool) -> Result, Error> { let conn = &mut get_conn(pool).await?; instance::table .inner_join(federation_blocklist::table) - .select(instance::domain) - .load::(conn) + .select(instance::all_columns) + .get_results(conn) .await } - pub async fn linked(pool: &DbPool) -> Result, Error> { + pub async fn linked(pool: &DbPool) -> Result, Error> { let conn = &mut get_conn(pool).await?; instance::table .left_join(federation_blocklist::table) .filter(federation_blocklist::id.is_null()) - .select(instance::domain) - .load::(conn) + .select(instance::all_columns) + .get_results(conn) .await } } diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 60152d6f..8c893cf9 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -648,6 +648,8 @@ table! { instance(id) { id -> Int4, domain -> Text, + software -> Nullable, + version -> Nullable, published -> Timestamp, updated -> Nullable, } diff --git a/crates/db_schema/src/source/instance.rs b/crates/db_schema/src/source/instance.rs index d5813005..a6c50cff 100644 --- a/crates/db_schema/src/source/instance.rs +++ b/crates/db_schema/src/source/instance.rs @@ -1,21 +1,30 @@ use crate::newtypes::InstanceId; #[cfg(feature = "full")] use crate::schema::instance; +use serde::{Deserialize, Serialize}; use std::fmt::Debug; +use typed_builder::TypedBuilder; -#[derive(PartialEq, Eq, Debug)] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[cfg_attr(feature = "full", derive(Queryable, Identifiable))] #[cfg_attr(feature = "full", diesel(table_name = instance))] pub struct Instance { pub id: InstanceId, pub domain: String, + pub software: Option, + pub version: Option, pub published: chrono::NaiveDateTime, pub updated: Option, } +#[derive(Clone, TypedBuilder)] +#[builder(field_defaults(default))] #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] #[cfg_attr(feature = "full", diesel(table_name = instance))] pub struct InstanceForm { + #[builder(!default)] pub domain: String, + pub software: Option, + pub version: Option, pub updated: Option, } diff --git a/crates/routes/src/nodeinfo.rs b/crates/routes/src/nodeinfo.rs index e1c70a87..72206e6d 100644 --- a/crates/routes/src/nodeinfo.rs +++ b/crates/routes/src/nodeinfo.rs @@ -34,27 +34,27 @@ async fn node_info(context: web::Data) -> Result, - pub usage: NodeInfoUsage, - pub open_registrations: bool, +#[derive(Serialize, Deserialize, Debug, Default)] +#[serde(rename_all = "camelCase", default)] +pub struct NodeInfo { + pub version: Option, + pub software: Option, + pub protocols: Option>, + pub usage: Option, + pub open_registrations: Option, } -#[derive(Serialize, Deserialize, Debug)] -struct NodeInfoSoftware { - pub name: String, - pub version: String, +#[derive(Serialize, Deserialize, Debug, Default)] +#[serde(default)] +pub struct NodeInfoSoftware { + pub name: Option, + pub version: Option, } -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct NodeInfoUsage { - pub users: NodeInfoUsers, - pub local_posts: i64, - pub local_comments: i64, +#[derive(Serialize, Deserialize, Debug, Default)] +#[serde(rename_all = "camelCase", default)] +pub struct NodeInfoUsage { + pub users: Option, + pub local_posts: Option, + pub local_comments: Option, } -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct NodeInfoUsers { - pub total: i64, - pub active_halfyear: i64, - pub active_month: i64, +#[derive(Serialize, Deserialize, Debug, Default)] +#[serde(rename_all = "camelCase", default)] +pub struct NodeInfoUsers { + pub total: Option, + pub active_halfyear: Option, + pub active_month: Option, } diff --git a/migrations/2023-02-13-221303_add_instance_software_and_version/down.sql b/migrations/2023-02-13-221303_add_instance_software_and_version/down.sql new file mode 100644 index 00000000..07179de1 --- /dev/null +++ b/migrations/2023-02-13-221303_add_instance_software_and_version/down.sql @@ -0,0 +1,2 @@ +alter table instance drop column software; +alter table instance drop column version; diff --git a/migrations/2023-02-13-221303_add_instance_software_and_version/up.sql b/migrations/2023-02-13-221303_add_instance_software_and_version/up.sql new file mode 100644 index 00000000..abfeb850 --- /dev/null +++ b/migrations/2023-02-13-221303_add_instance_software_and_version/up.sql @@ -0,0 +1,4 @@ +-- Add Software and Version columns from nodeinfo to the instance table + +alter table instance add column software varchar(255); +alter table instance add column version varchar(255); diff --git a/scripts/fix-clippy.sh b/scripts/fix-clippy.sh index 8de66015..6b8be28b 100755 --- a/scripts/fix-clippy.sh +++ b/scripts/fix-clippy.sh @@ -1,12 +1,18 @@ #!/bin/bash set -e +CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +cd $CWD/../ + cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- \ - -D warnings -D deprecated -D clippy::perf -D clippy::complexity \ - -D clippy::style -D clippy::correctness -D clippy::suspicious \ - -D clippy::dbg_macro -D clippy::inefficient_to_string \ - -D clippy::items-after-statements -D clippy::implicit_clone \ - -D clippy::wildcard_imports -D clippy::cast_lossless \ - -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \ - -D clippy::unused_self \ - -A clippy::uninlined_format_args + -D warnings -D deprecated -D clippy::perf -D clippy::complexity \ + -D clippy::style -D clippy::correctness -D clippy::suspicious \ + -D clippy::dbg_macro -D clippy::inefficient_to_string \ + -D clippy::items-after-statements -D clippy::implicit_clone \ + -D clippy::wildcard_imports -D clippy::cast_lossless \ + -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \ + -D clippy::unused_self \ + -A clippy::uninlined_format_args + +cargo +nightly fmt diff --git a/src/lib.rs b/src/lib.rs index 3c6b283e..3605f88c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,7 +42,7 @@ use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry} use url::Url; /// Max timeout for http requests -const REQWEST_TIMEOUT: Duration = Duration::from_secs(10); +pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10); /// Placing the main function in lib.rs allows other crates to import it and embed Lemmy pub async fn start_lemmy_server() -> Result<(), LemmyError> { @@ -73,11 +73,6 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { let pool = build_db_pool(&settings).await?; run_advanced_migrations(&pool, &settings).await?; - // Schedules various cleanup tasks for the DB - thread::spawn(move || { - scheduled_tasks::setup(db_url).expect("Couldn't set up scheduled_tasks"); - }); - // Initialize the secrets let secret = Secret::init(&pool) .await @@ -106,8 +101,9 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { settings.bind, settings.port ); + let user_agent = build_user_agent(&settings); let reqwest_client = Client::builder() - .user_agent(build_user_agent(&settings)) + .user_agent(user_agent.clone()) .timeout(REQWEST_TIMEOUT) .build()?; @@ -128,6 +124,11 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { .with(TracingMiddleware::default()) .build(); + // Schedules various cleanup tasks for the DB + thread::spawn(move || { + scheduled_tasks::setup(db_url, user_agent).expect("Couldn't set up scheduled_tasks"); + }); + let chat_server = Arc::new(ChatServer::startup()); // Create Http server with websocket support diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 57854911..4fc85b8f 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -2,17 +2,25 @@ use clokwerk::{Scheduler, TimeUnits}; // Import week days and WeekDay use diesel::{sql_query, PgConnection, RunQueryDsl}; use diesel::{Connection, ExpressionMethods, QueryDsl}; -use lemmy_utils::error::LemmyError; +use lemmy_db_schema::{ + source::instance::{Instance, InstanceForm}, + utils::naive_now, +}; +use lemmy_routes::nodeinfo::NodeInfo; +use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT}; +use reqwest::blocking::Client; use std::{thread, time::Duration}; use tracing::info; /// Schedules various cleanup tasks for lemmy in a background thread -pub fn setup(db_url: String) -> Result<(), LemmyError> { +pub fn setup(db_url: String, user_agent: String) -> Result<(), LemmyError> { // Setup the connections let mut scheduler = Scheduler::new(); let mut conn = PgConnection::establish(&db_url).expect("could not establish connection"); + let mut conn_2 = PgConnection::establish(&db_url).expect("could not establish connection"); + active_counts(&mut conn); update_banned_when_expired(&mut conn); @@ -33,6 +41,11 @@ pub fn setup(db_url: String) -> Result<(), LemmyError> { clear_old_activities(&mut conn); }); + update_instance_software(&mut conn_2, &user_agent); + scheduler.every(1.days()).run(move || { + update_instance_software(&mut conn_2, &user_agent); + }); + // Manually run the scheduler in an event loop loop { scheduler.run_pending(); @@ -120,3 +133,67 @@ fn drop_ccnew_indexes(conn: &mut PgConnection) { .execute(conn) .expect("drop ccnew indexes"); } + +/// Updates the instance software and version +fn update_instance_software(conn: &mut PgConnection, user_agent: &str) { + use lemmy_db_schema::schema::instance; + info!("Updating instances software and versions..."); + + let client = Client::builder() + .user_agent(user_agent) + .timeout(REQWEST_TIMEOUT) + .build() + .expect("couldnt build reqwest client"); + + let instances = instance::table + .get_results::(conn) + .expect("no instances found"); + + for instance in instances { + let node_info_url = format!("https://{}/nodeinfo/2.0.json", instance.domain); + + // Skip it if it can't connect + let res = client + .get(&node_info_url) + .send() + .ok() + .and_then(|t| t.json::().ok()); + + if let Some(node_info) = res { + let software = node_info.software.as_ref(); + let form = InstanceForm::builder() + .domain(instance.domain) + .software(software.and_then(|s| s.name.clone())) + .version(software.and_then(|s| s.version.clone())) + .updated(Some(naive_now())) + .build(); + + diesel::update(instance::table.find(instance.id)) + .set(form) + .execute(conn) + .expect("update site instance software"); + } + } + info!("Done."); +} + +#[cfg(test)] +mod tests { + use lemmy_routes::nodeinfo::NodeInfo; + use reqwest::Client; + + #[tokio::test] + async fn test_nodeinfo() { + let client = Client::builder().build().unwrap(); + let lemmy_ml_nodeinfo = client + .get("https://lemmy.ml/nodeinfo/2.0.json") + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + + assert_eq!(lemmy_ml_nodeinfo.software.unwrap().name.unwrap(), "lemmy"); + } +} -- 2.44.1