tracing-log = "0.1.3"
tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
url = { version = "2.3.1", features = ["serde"] }
-reqwest = { version = "0.11.12", features = ["json"] }
+reqwest = { version = "0.11.12", features = ["json", "blocking"] }
reqwest-middleware = "0.2.0"
reqwest-tracing = "0.4.0"
clokwerk = "0.3.5"
use crate::sensitive::Sensitive;
use lemmy_db_schema::{
newtypes::{CommentId, CommunityId, LanguageId, PersonId, PostId},
- source::{language::Language, local_site::RegistrationMode, tagline::Tagline},
+ source::{
+ instance::Instance,
+ language::Language,
+ local_site::RegistrationMode,
+ tagline::Tagline,
+ },
ListingType,
ModlogActionType,
SearchType,
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct FederatedInstances {
- pub linked: Vec<String>,
- pub allowed: Option<Vec<String>>,
- pub blocked: Option<Vec<String>>,
+ pub linked: Vec<Instance>,
+ pub allowed: Option<Vec<Instance>>,
+ pub blocked: Option<Vec<Instance>>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
}
if let Some(blocked) = local_site_data.blocked_instances.as_ref() {
- if blocked.contains(&domain) {
+ if blocked.iter().any(|i| domain.eq(&i.domain)) {
return Err("Domain is blocked");
}
}
if let Some(allowed) = local_site_data.allowed_instances.as_ref() {
- if !allowed.contains(&domain) {
+ if !allowed.iter().any(|i| domain.eq(&i.domain)) {
return Err("Domain is not in allowlist");
}
}
#[derive(Clone)]
pub(crate) struct LocalSiteData {
local_site: Option<LocalSite>,
- allowed_instances: Option<Vec<String>>,
- blocked_instances: Option<Vec<String>>,
+ allowed_instances: Option<Vec<Instance>>,
+ blocked_instances: Option<Vec<Instance>>,
}
pub(crate) async fn fetch_local_site_data(
if is_strict {
// need to allow this explicitly because apub receive might contain objects from our local
// instance.
- let mut allowed_and_local = allowed.clone();
+ let mut allowed_and_local = allowed
+ .iter()
+ .map(|i| i.domain.clone())
+ .collect::<Vec<String>>();
allowed_and_local.push(local_instance);
if !allowed_and_local.contains(&domain) {
#[serial]
async fn test_allowlist_insert_and_clear() {
let pool = &build_db_pool_for_tests().await;
- let allowed = Some(vec![
+ let domains = vec![
"tld1.xyz".to_string(),
"tld2.xyz".to_string(),
"tld3.xyz".to_string(),
- ]);
+ ];
+
+ let allowed = Some(domains.clone());
FederationAllowList::replace(pool, allowed).await.unwrap();
let allows = Instance::allowlist(pool).await.unwrap();
+ let allows_domains = allows
+ .iter()
+ .map(|i| i.domain.clone())
+ .collect::<Vec<String>>();
assert_eq!(3, allows.len());
- assert_eq!(
- vec![
- "tld1.xyz".to_string(),
- "tld2.xyz".to_string(),
- "tld3.xyz".to_string()
- ],
- allows
- );
+ assert_eq!(domains, allows_domains);
// Now test clearing them via Some(empty vec)
let clear_allows = Some(Vec::new());
Self::create(pool, domain).await
}
pub async fn create_conn(conn: &mut AsyncPgConnection, domain: &str) -> Result<Self, Error> {
- let form = InstanceForm {
- domain: domain.to_string(),
- updated: Some(naive_now()),
- };
+ let form = InstanceForm::builder()
+ .domain(domain.to_string())
+ .updated(Some(naive_now()))
+ .build();
Self::create_from_form_conn(conn, &form).await
}
pub async fn delete(pool: &DbPool, instance_id: InstanceId) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
diesel::delete(instance::table).execute(conn).await
}
- pub async fn allowlist(pool: &DbPool) -> Result<Vec<String>, Error> {
+ pub async fn allowlist(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
instance::table
.inner_join(federation_allowlist::table)
- .select(instance::domain)
- .load::<String>(conn)
+ .select(instance::all_columns)
+ .get_results(conn)
.await
}
- pub async fn blocklist(pool: &DbPool) -> Result<Vec<String>, Error> {
+ pub async fn blocklist(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
instance::table
.inner_join(federation_blocklist::table)
- .select(instance::domain)
- .load::<String>(conn)
+ .select(instance::all_columns)
+ .get_results(conn)
.await
}
- pub async fn linked(pool: &DbPool) -> Result<Vec<String>, Error> {
+ pub async fn linked(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
instance::table
.left_join(federation_blocklist::table)
.filter(federation_blocklist::id.is_null())
- .select(instance::domain)
- .load::<String>(conn)
+ .select(instance::all_columns)
+ .get_results(conn)
.await
}
}
instance(id) {
id -> Int4,
domain -> Text,
+ software -> Nullable<Text>,
+ version -> Nullable<Text>,
published -> Timestamp,
updated -> Nullable<Timestamp>,
}
use crate::newtypes::InstanceId;
#[cfg(feature = "full")]
use crate::schema::instance;
+use serde::{Deserialize, Serialize};
use std::fmt::Debug;
+use typed_builder::TypedBuilder;
-#[derive(PartialEq, Eq, Debug)]
+#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = instance))]
pub struct Instance {
pub id: InstanceId,
pub domain: String,
+ pub software: Option<String>,
+ pub version: Option<String>,
pub published: chrono::NaiveDateTime,
pub updated: Option<chrono::NaiveDateTime>,
}
+#[derive(Clone, TypedBuilder)]
+#[builder(field_defaults(default))]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = instance))]
pub struct InstanceForm {
+ #[builder(!default)]
pub domain: String,
+ pub software: Option<String>,
+ pub version: Option<String>,
pub updated: Option<chrono::NaiveDateTime>,
}
.map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?;
let protocols = if site_view.local_site.federation_enabled {
- vec!["activitypub".to_string()]
+ Some(vec!["activitypub".to_string()])
} else {
- vec![]
+ None
};
- let open_registrations = site_view.local_site.registration_mode == RegistrationMode::Open;
+ let open_registrations = Some(site_view.local_site.registration_mode == RegistrationMode::Open);
let json = NodeInfo {
- version: "2.0".to_string(),
- software: NodeInfoSoftware {
- name: "lemmy".to_string(),
- version: version::VERSION.to_string(),
- },
+ version: Some("2.0".to_string()),
+ software: Some(NodeInfoSoftware {
+ name: Some("lemmy".to_string()),
+ version: Some(version::VERSION.to_string()),
+ }),
protocols,
- usage: NodeInfoUsage {
- users: NodeInfoUsers {
- total: site_view.counts.users,
- active_halfyear: site_view.counts.users_active_half_year,
- active_month: site_view.counts.users_active_month,
- },
- local_posts: site_view.counts.posts,
- local_comments: site_view.counts.comments,
- },
+ usage: Some(NodeInfoUsage {
+ users: Some(NodeInfoUsers {
+ total: Some(site_view.counts.users),
+ active_halfyear: Some(site_view.counts.users_active_half_year),
+ active_month: Some(site_view.counts.users_active_month),
+ }),
+ local_posts: Some(site_view.counts.posts),
+ local_comments: Some(site_view.counts.comments),
+ }),
open_registrations,
};
pub href: Url,
}
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-struct NodeInfo {
- pub version: String,
- pub software: NodeInfoSoftware,
- pub protocols: Vec<String>,
- pub usage: NodeInfoUsage,
- pub open_registrations: bool,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(rename_all = "camelCase", default)]
+pub struct NodeInfo {
+ pub version: Option<String>,
+ pub software: Option<NodeInfoSoftware>,
+ pub protocols: Option<Vec<String>>,
+ pub usage: Option<NodeInfoUsage>,
+ pub open_registrations: Option<bool>,
}
-#[derive(Serialize, Deserialize, Debug)]
-struct NodeInfoSoftware {
- pub name: String,
- pub version: String,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(default)]
+pub struct NodeInfoSoftware {
+ pub name: Option<String>,
+ pub version: Option<String>,
}
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-struct NodeInfoUsage {
- pub users: NodeInfoUsers,
- pub local_posts: i64,
- pub local_comments: i64,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(rename_all = "camelCase", default)]
+pub struct NodeInfoUsage {
+ pub users: Option<NodeInfoUsers>,
+ pub local_posts: Option<i64>,
+ pub local_comments: Option<i64>,
}
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-struct NodeInfoUsers {
- pub total: i64,
- pub active_halfyear: i64,
- pub active_month: i64,
+#[derive(Serialize, Deserialize, Debug, Default)]
+#[serde(rename_all = "camelCase", default)]
+pub struct NodeInfoUsers {
+ pub total: Option<i64>,
+ pub active_halfyear: Option<i64>,
+ pub active_month: Option<i64>,
}
--- /dev/null
+alter table instance drop column software;
+alter table instance drop column version;
--- /dev/null
+-- Add Software and Version columns from nodeinfo to the instance table
+
+alter table instance add column software varchar(255);
+alter table instance add column version varchar(255);
#!/bin/bash
set -e
+CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+
+cd $CWD/../
+
cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- \
- -D warnings -D deprecated -D clippy::perf -D clippy::complexity \
- -D clippy::style -D clippy::correctness -D clippy::suspicious \
- -D clippy::dbg_macro -D clippy::inefficient_to_string \
- -D clippy::items-after-statements -D clippy::implicit_clone \
- -D clippy::wildcard_imports -D clippy::cast_lossless \
- -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \
- -D clippy::unused_self \
- -A clippy::uninlined_format_args
+ -D warnings -D deprecated -D clippy::perf -D clippy::complexity \
+ -D clippy::style -D clippy::correctness -D clippy::suspicious \
+ -D clippy::dbg_macro -D clippy::inefficient_to_string \
+ -D clippy::items-after-statements -D clippy::implicit_clone \
+ -D clippy::wildcard_imports -D clippy::cast_lossless \
+ -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \
+ -D clippy::unused_self \
+ -A clippy::uninlined_format_args
+
+cargo +nightly fmt
use url::Url;
/// Max timeout for http requests
-const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
+pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
/// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
pub async fn start_lemmy_server() -> Result<(), LemmyError> {
let pool = build_db_pool(&settings).await?;
run_advanced_migrations(&pool, &settings).await?;
- // Schedules various cleanup tasks for the DB
- thread::spawn(move || {
- scheduled_tasks::setup(db_url).expect("Couldn't set up scheduled_tasks");
- });
-
// Initialize the secrets
let secret = Secret::init(&pool)
.await
settings.bind, settings.port
);
+ let user_agent = build_user_agent(&settings);
let reqwest_client = Client::builder()
- .user_agent(build_user_agent(&settings))
+ .user_agent(user_agent.clone())
.timeout(REQWEST_TIMEOUT)
.build()?;
.with(TracingMiddleware::default())
.build();
+ // Schedules various cleanup tasks for the DB
+ thread::spawn(move || {
+ scheduled_tasks::setup(db_url, user_agent).expect("Couldn't set up scheduled_tasks");
+ });
+
let chat_server = Arc::new(ChatServer::startup());
// Create Http server with websocket support
// Import week days and WeekDay
use diesel::{sql_query, PgConnection, RunQueryDsl};
use diesel::{Connection, ExpressionMethods, QueryDsl};
-use lemmy_utils::error::LemmyError;
+use lemmy_db_schema::{
+ source::instance::{Instance, InstanceForm},
+ utils::naive_now,
+};
+use lemmy_routes::nodeinfo::NodeInfo;
+use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT};
+use reqwest::blocking::Client;
use std::{thread, time::Duration};
use tracing::info;
/// Schedules various cleanup tasks for lemmy in a background thread
-pub fn setup(db_url: String) -> Result<(), LemmyError> {
+pub fn setup(db_url: String, user_agent: String) -> Result<(), LemmyError> {
// Setup the connections
let mut scheduler = Scheduler::new();
let mut conn = PgConnection::establish(&db_url).expect("could not establish connection");
+ let mut conn_2 = PgConnection::establish(&db_url).expect("could not establish connection");
+
active_counts(&mut conn);
update_banned_when_expired(&mut conn);
clear_old_activities(&mut conn);
});
+ update_instance_software(&mut conn_2, &user_agent);
+ scheduler.every(1.days()).run(move || {
+ update_instance_software(&mut conn_2, &user_agent);
+ });
+
// Manually run the scheduler in an event loop
loop {
scheduler.run_pending();
.execute(conn)
.expect("drop ccnew indexes");
}
+
+/// Updates the instance software and version
+fn update_instance_software(conn: &mut PgConnection, user_agent: &str) {
+ use lemmy_db_schema::schema::instance;
+ info!("Updating instances software and versions...");
+
+ let client = Client::builder()
+ .user_agent(user_agent)
+ .timeout(REQWEST_TIMEOUT)
+ .build()
+ .expect("couldnt build reqwest client");
+
+ let instances = instance::table
+ .get_results::<Instance>(conn)
+ .expect("no instances found");
+
+ for instance in instances {
+ let node_info_url = format!("https://{}/nodeinfo/2.0.json", instance.domain);
+
+ // Skip it if it can't connect
+ let res = client
+ .get(&node_info_url)
+ .send()
+ .ok()
+ .and_then(|t| t.json::<NodeInfo>().ok());
+
+ if let Some(node_info) = res {
+ let software = node_info.software.as_ref();
+ let form = InstanceForm::builder()
+ .domain(instance.domain)
+ .software(software.and_then(|s| s.name.clone()))
+ .version(software.and_then(|s| s.version.clone()))
+ .updated(Some(naive_now()))
+ .build();
+
+ diesel::update(instance::table.find(instance.id))
+ .set(form)
+ .execute(conn)
+ .expect("update site instance software");
+ }
+ }
+ info!("Done.");
+}
+
+#[cfg(test)]
+mod tests {
+ use lemmy_routes::nodeinfo::NodeInfo;
+ use reqwest::Client;
+
+ #[tokio::test]
+ async fn test_nodeinfo() {
+ let client = Client::builder().build().unwrap();
+ let lemmy_ml_nodeinfo = client
+ .get("https://lemmy.ml/nodeinfo/2.0.json")
+ .send()
+ .await
+ .unwrap()
+ .json::<NodeInfo>()
+ .await
+ .unwrap();
+
+ assert_eq!(lemmy_ml_nodeinfo.software.unwrap().name.unwrap(), "lemmy");
+ }
+}
projects / lemmy.git / commitdiff