]> Untitled Git - lemmy.git/commitdiff
Proxy pictrs requests through Lemmy (fixes #371) (#77)
authornutomic <nutomic@noreply.yerbamate.dev>
Wed, 5 Aug 2020 16:00:00 +0000 (16:00 +0000)
committernutomic <nutomic@noreply.yerbamate.dev>
Wed, 5 Aug 2020 16:00:00 +0000 (16:00 +0000)
fix check_only value for image rate limit

Fix image rate limit

Add rate limit for image uploads

Proxy pictrs requests through Lemmy (fixes #371)

Co-authored-by: Felix Ableitner <me@nutomic.com>
Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/77

ansible/templates/nginx.conf
docker/dev/docker-compose.yml
docker/federation/nginx.conf
docker/lemmy.hjson
server/config/defaults.hjson
server/lemmy_utils/src/settings.rs
server/src/main.rs
server/src/rate_limit/mod.rs
server/src/rate_limit/rate_limiter.rs
server/src/routes/images.rs [new file with mode: 0644]
server/src/routes/mod.rs

index 4f66292c367a4d2d382bde702c7f1bc815a36021..092f855203ad8951d1fa877a7d4985cfd058646f 100644 (file)
@@ -74,18 +74,6 @@ server {
       return 301 /pictrs/image/$1;
     }
 
-    # pict-rs images
-    location /pictrs {
-      location /pictrs/image {
-        proxy_pass http://0.0.0.0:8537/image;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      }
-      # Block the import
-      return 403;
-    }
-
     location /iframely/ {
       proxy_pass http://0.0.0.0:8061/;
       proxy_set_header X-Real-IP $remote_addr;
index 51a3ecdab307bb219f23db28d61de440ebd698fd..257ad6c63b0eb44421cee47b581dfbff8e7713c4 100644 (file)
@@ -21,7 +21,8 @@ services:
   postgres:
     image: postgres:12-alpine
     ports:
-      - "127.0.0.1:5432:5432"
+      # use a different port so it doesnt conflict with postgres running on the host
+      - "127.0.0.1:5433:5432"
     environment:
       - POSTGRES_USER=lemmy
       - POSTGRES_PASSWORD=password
index 573067981ae42661c6a5f693fe83578335f2e9a5..b7901c19cf67c7dfd51123b2b4d09c9f322830b6 100644 (file)
@@ -26,18 +26,6 @@ http {
             proxy_set_header Connection "upgrade";
         }
 
-        # pict-rs images
-        location /pictrs {
-          location /pictrs/image {
-            proxy_pass http://pictrs:8080/image;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          }
-          # Block the import
-          return 403;
-        }
-
         location /iframely/ {
             proxy_pass http://iframely:80/;
             proxy_set_header X-Real-IP $remote_addr;
@@ -69,18 +57,6 @@ http {
             proxy_set_header Connection "upgrade";
         }
 
-        # pict-rs images
-        location /pictrs {
-          location /pictrs/image {
-            proxy_pass http://pictrs:8080/image;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          }
-          # Block the import
-          return 403;
-        }
-
         location /iframely/ {
             proxy_pass http://iframely:80/;
             proxy_set_header X-Real-IP $remote_addr;
@@ -112,18 +88,6 @@ http {
             proxy_set_header Connection "upgrade";
         }
 
-        # pict-rs images
-        location /pictrs {
-          location /pictrs/image {
-            proxy_pass http://pictrs:8080/image;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          }
-          # Block the import
-          return 403;
-        }
-
         location /iframely/ {
             proxy_pass http://iframely:80/;
             proxy_set_header X-Real-IP $remote_addr;
index 89da46891364e07efa3cc23614abbda26462b299..d17394767433b018e75563ff8f938873a3100a70 100644 (file)
@@ -2,6 +2,15 @@
   # for more info about the config, check out the documentation
   # https://dev.lemmy.ml/docs/administration_configuration.html
 
+  setup: {
+    # username for the admin user
+    admin_username: "lemmy"
+    # password for the admin user
+    admin_password: "lemmy"
+    # name of the site (can be changed later)
+    site_name: "lemmy-test"
+  }
+
   # the domain name of your instance (eg "dev.lemmy.ml")
   hostname: "my_domain"
   # address where lemmy should listen for incoming requests
index 5238455a7136a6cbc83abc6135393f8144ab5f9f..9e9fc998818c899a86d6c940c7703345f752c6a4 100644 (file)
@@ -35,6 +35,8 @@
   jwt_secret: "changeme"
   # The location of the frontend
   front_end_dir: "../ui/dist"
+  # address where pictrs is available
+  pictrs_url: "http://pictrs:8080"
   # rate limits for various user actions, by user ip
   rate_limit: {
     # maximum number of messages created in interval
     register: 3
     # interval length for registration limit
     register_per_second: 3600
+    # maximum number of image uploads in interval
+    image: 6
+    # interval length for image uploads
+    image_per_second: 3600
   }
   # settings related to activitypub federation
   federation: {
index b7cc2c45f5ecdb26443d10a1477eeea542bfd704..6a566de7ef2b38afd341fcf73f158fda2a208362 100644 (file)
@@ -14,6 +14,7 @@ pub struct Settings {
   pub port: u16,
   pub jwt_secret: String,
   pub front_end_dir: String,
+  pub pictrs_url: String,
   pub rate_limit: RateLimitConfig,
   pub email: Option<EmailConfig>,
   pub federation: Federation,
@@ -36,6 +37,8 @@ pub struct RateLimitConfig {
   pub post_per_second: i32,
   pub register: i32,
   pub register_per_second: i32,
+  pub image: i32,
+  pub image_per_second: i32,
 }
 
 #[derive(Debug, Deserialize, Clone)]
index 7689d7ad1aa363e87468ce6a0e5ff2870cffafca..b27ddb9cbcc7fe4359613bbce6ef15fe64186aa5 100644 (file)
@@ -27,7 +27,7 @@ use lemmy_server::{
   blocking,
   code_migrations::run_advanced_migrations,
   rate_limit::{rate_limiter::RateLimiter, RateLimit},
-  routes::{api, federation, feeds, index, nodeinfo, webfinger},
+  routes::*,
   websocket::server::*,
   LemmyError,
 };
@@ -91,9 +91,10 @@ async fn main() -> Result<(), LemmyError> {
       .data(server.clone())
       .data(Client::default())
       // The routes
-      .configure(move |cfg| api::config(cfg, &rate_limiter))
+      .configure(|cfg| api::config(cfg, &rate_limiter))
       .configure(federation::config)
       .configure(feeds::config)
+      .configure(|cfg| images::config(cfg, &rate_limiter))
       .configure(index::config)
       .configure(nodeinfo::config)
       .configure(webfinger::config)
index 513c923c6182b006ebedff3cf8c57ac1cbf63f0b..39df726505c02ea03915120747604c003b506f3c 100644 (file)
@@ -45,6 +45,10 @@ impl RateLimit {
     self.kind(RateLimitType::Register)
   }
 
+  pub fn image(&self) -> RateLimited {
+    self.kind(RateLimitType::Image)
+  }
+
   fn kind(&self, type_: RateLimitType) -> RateLimited {
     RateLimited {
       rate_limiter: self.rate_limiter.clone(),
@@ -101,6 +105,15 @@ impl RateLimited {
             true,
           )?;
         }
+        RateLimitType::Image => {
+          limiter.check_rate_limit_full(
+            self.type_,
+            &ip_addr,
+            rate_limit.image,
+            rate_limit.image_per_second,
+            false,
+          )?;
+        }
       };
     }
 
index 20a617c2fe3097d4617750ed8cd0460df3c87f9c..f1a38841244c4a5c429a03945825a5dcdfb4ed4e 100644 (file)
@@ -15,6 +15,7 @@ pub enum RateLimitType {
   Message,
   Register,
   Post,
+  Image,
 }
 
 /// Rate limiting based on rate type and IP addr
diff --git a/server/src/routes/images.rs b/server/src/routes/images.rs
new file mode 100644 (file)
index 0000000..8c94535
--- /dev/null
@@ -0,0 +1,146 @@
+use crate::rate_limit::RateLimit;
+use actix::clock::Duration;
+use actix_web::{body::BodyStream, http::StatusCode, *};
+use awc::Client;
+use lemmy_utils::settings::Settings;
+use serde::{Deserialize, Serialize};
+
+const THUMBNAIL_SIZES: &[u64] = &[256];
+
+pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) {
+  let client = Client::build()
+    .header("User-Agent", "pict-rs-frontend, v0.1.0")
+    .timeout(Duration::from_secs(30))
+    .finish();
+
+  cfg
+    .data(client)
+    .service(
+      web::resource("/pictrs/image")
+        .wrap(rate_limit.image())
+        .route(web::post().to(upload)),
+    )
+    .service(web::resource("/pictrs/image/{filename}").route(web::get().to(full_res)))
+    .service(
+      web::resource("/pictrs/image/thumbnail{size}/{filename}").route(web::get().to(thumbnail)),
+    )
+    .service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete)));
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Image {
+  file: String,
+  delete_token: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Images {
+  msg: String,
+  files: Option<Vec<Image>>,
+}
+
+async fn upload(
+  req: HttpRequest,
+  body: web::Payload,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  // TODO: check auth and rate limit here
+
+  let mut res = client
+    .request_from(format!("{}/image", Settings::get().pictrs_url), req.head())
+    .if_some(req.head().peer_addr, |addr, req| {
+      req.header("X-Forwarded-For", addr.to_string())
+    })
+    .send_stream(body)
+    .await?;
+
+  let images = res.json::<Images>().await?;
+
+  Ok(HttpResponse::build(res.status()).json(images))
+}
+
+async fn full_res(
+  filename: web::Path<String>,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let url = format!(
+    "{}/image/{}",
+    Settings::get().pictrs_url,
+    &filename.into_inner()
+  );
+  image(url, req, client).await
+}
+
+async fn thumbnail(
+  parts: web::Path<(u64, String)>,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let (size, file) = parts.into_inner();
+
+  if THUMBNAIL_SIZES.contains(&size) {
+    let url = format!(
+      "{}/image/thumbnail{}/{}",
+      Settings::get().pictrs_url,
+      size,
+      &file
+    );
+
+    return image(url, req, client).await;
+  }
+
+  Ok(HttpResponse::NotFound().finish())
+}
+
+async fn image(
+  url: String,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let res = client
+    .request_from(url, req.head())
+    .if_some(req.head().peer_addr, |addr, req| {
+      req.header("X-Forwarded-For", addr.to_string())
+    })
+    .no_decompress()
+    .send()
+    .await?;
+
+  if res.status() == StatusCode::NOT_FOUND {
+    return Ok(HttpResponse::NotFound().finish());
+  }
+
+  let mut client_res = HttpResponse::build(res.status());
+
+  for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") {
+    client_res.header(name.clone(), value.clone());
+  }
+
+  Ok(client_res.body(BodyStream::new(res)))
+}
+
+async fn delete(
+  components: web::Path<(String, String)>,
+  req: HttpRequest,
+  client: web::Data<Client>,
+) -> Result<HttpResponse, Error> {
+  let (token, file) = components.into_inner();
+
+  let url = format!(
+    "{}/image/delete/{}/{}",
+    Settings::get().pictrs_url,
+    &token,
+    &file
+  );
+  let res = client
+    .request_from(url, req.head())
+    .if_some(req.head().peer_addr, |addr, req| {
+      req.header("X-Forwarded-For", addr.to_string())
+    })
+    .no_decompress()
+    .send()
+    .await?;
+
+  Ok(HttpResponse::build(res.status()).body(BodyStream::new(res)))
+}
index bcb7e45fa33bd631191838008c070b1a839194d7..4a7d30993f6a849a4b690b39beb2d2a75db23ddf 100644 (file)
@@ -1,6 +1,7 @@
 pub mod api;
 pub mod federation;
 pub mod feeds;
+pub mod images;
 pub mod index;
 pub mod nodeinfo;
 pub mod webfinger;