Dont pass accept-encoding header to pictrs (ref #1734) (#1738)

* Allow image upload in docker/dev/ (replaces #1765)

* Dont pass accept-encoding header to pictrs (ref #1734)

* Fix clippy errors

Cargo.lock

@@ -70,12 +70,10 @@ dependencies = [
  "ahash",
  "base64 0.13.0",
  "bitflags",
- "brotli2",
  "bytes",
  "bytestring",
  "derive_more",
  "encoding_rs",
- "flate2",
  "futures-core",
  "futures-util",
  "h2",
@@ -97,7 +95,6 @@ dependencies = [
  "smallvec",
  "time 0.2.27",
  "tokio",
- "zstd",
 ]
 
 [[package]]
@@ -378,7 +375,6 @@ dependencies = [
  "base64 0.13.0",
  "bytes",
  "cfg-if",
- "cookie",
  "derive_more",
  "futures-core",
  "itoa",
@@ -530,26 +526,6 @@ dependencies = [
  "opaque-debug 0.3.0",
 ]
 
-[[package]]
-name = "brotli-sys"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd"
-dependencies = [
- "cc",
- "libc",
-]
-
-[[package]]
-name = "brotli2"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e"
-dependencies = [
- "brotli-sys",
- "libc",
-]
-
 [[package]]
 name = "bumpalo"
 version = "3.7.0"
@@ -614,9 +590,6 @@ name = "cc"
 version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
-dependencies = [
- "jobserver",
-]
 
 [[package]]
 name = "cfg-if"
@@ -1559,15 +1532,6 @@ version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
-[[package]]
-name = "jobserver"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "jpeg-decoder"
 version = "0.1.22"
@@ -1867,6 +1831,7 @@ name = "lemmy_routes"
 version = "0.12.2-rc.1"
 dependencies = [
  "actix",
+ "actix-http",
  "actix-web",
  "actix-web-actors",
  "anyhow",
@@ -4001,32 +3966,3 @@ name = "xdg"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57"
-
-[[package]]
-name = "zstd"
-version = "0.7.0+zstd.1.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52"
-dependencies = [
- "zstd-safe",
-]
-
-[[package]]
-name = "zstd-safe"
-version = "3.1.0+zstd.1.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d"
-dependencies = [
- "libc",
- "zstd-sys",
-]
-
-[[package]]
-name = "zstd-sys"
-version = "1.5.0+zstd.1.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465"
-dependencies = [
- "cc",
- "libc",
-]

crates/routes/Cargo.toml

@@ -21,15 +21,14 @@ diesel = "1.4.7"
 actix = "0.12.0"
 actix-web = { version = "4.0.0-beta.8", default-features = false, features = ["rustls"] }
 actix-web-actors = { version = "4.0.0-beta.6", default-features = false }
+actix-http = "3.0.0-beta.9"
 sha2 = "0.9.5"
 log = "0.4.14"
 anyhow = "1.0.43"
 chrono = { version = "0.4.19", features = ["serde"] }
 rss = "1.10.0"
 serde = { version = "1.0.129", features = ["derive"] }
-# TODO awc should use default-features = false, because gzip is a heavy dependency.
+awc = { version  = "3.0.0-beta.7", default-features = false }
-# This must wait for pictrs to have a configurable disabled gzip
-awc = "3.0.0-beta.7"
 url = { version = "2.2.2", features = ["serde"] }
 strum = "0.21.0"
 lazy_static = "1.4.0"

crates/routes/src/images.rs

@@ -1,3 +1,4 @@
+use actix_http::http::header::ACCEPT_ENCODING;
 use actix_web::{body::BodyStream, http::StatusCode, web::Data, *};
 use anyhow::anyhow;
 use awc::Client;
@@ -56,6 +57,8 @@ async fn upload(
   };
 
   let mut client_req = client.request_from(format!("{}/image", pictrs_url()?), req.head());
+  // remove content-encoding header so that pictrs doesnt send gzipped response
+  client_req.headers_mut().remove(ACCEPT_ENCODING);
 
   if let Some(addr) = req.head().peer_addr {
     client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string()))
@@ -103,6 +106,7 @@ async fn image(
   client: web::Data<Client>,
 ) -> Result<HttpResponse, Error> {
   let mut client_req = client.request_from(url, req.head());
+  client_req.headers_mut().remove(ACCEPT_ENCODING);
 
   if let Some(addr) = req.head().peer_addr {
     client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string()))
@@ -137,6 +141,7 @@ async fn delete(
   let url = format!("{}/image/delete/{}/{}", pictrs_url()?, &token, &file);
 
   let mut client_req = client.request_from(url, req.head());
+  client_req.headers_mut().remove(ACCEPT_ENCODING);
 
   if let Some(addr) = req.head().peer_addr {
     client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string()))

crates/utils/src/rate_limit/rate_limiter.rs

@@ -18,19 +18,11 @@ pub(crate) enum RateLimitType {
 }
 
 /// Rate limiting based on rate type and IP addr
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 pub struct RateLimiter {
   buckets: HashMap<RateLimitType, HashMap<IpAddr, RateLimitBucket>>,
 }
 
-impl Default for RateLimiter {
-  fn default() -> Self {
-    Self {
-      buckets: HashMap::<RateLimitType, HashMap<IpAddr, RateLimitBucket>>::new(),
-    }
-  }
-}
-
 impl RateLimiter {
   fn insert_ip(&mut self, ip: &IpAddr) {
     for rate_limit_type in RateLimitType::iter() {

crates/utils/src/request.rs

@@ -114,6 +114,7 @@ pub(crate) struct PictrsResponse {
 #[derive(Deserialize, Debug, Clone)]
 pub(crate) struct PictrsFile {
   file: String,
+  #[allow(dead_code)]
   delete_token: String,
 }
 

docker/dev/docker-compose.yml

@@ -1,6 +1,16 @@
 version: '3.3'
 
 services:
+  nginx:
+    image: nginx:1-alpine
+    ports:
+      - "1236:1236"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+    restart: always
+    depends_on:
+      - pictrs
+      - lemmy-ui
 
   lemmy:
     image: lemmy-dev:latest
@@ -17,12 +27,10 @@ services:
 
   lemmy-ui:
     image: dessalines/lemmy-ui:dev
-    ports:
-      - "1235:1234"
     restart: always
     environment:
       - LEMMY_INTERNAL_HOST=lemmy:8536
-      - LEMMY_EXTERNAL_HOST=localhost:8536
+      - LEMMY_EXTERNAL_HOST=localhost:1234
       - LEMMY_HTTPS=false
     depends_on: 
       - lemmy
@@ -42,8 +50,6 @@ services:
 
   pictrs:
     image: asonix/pictrs:v0.2.6-r2
-    ports: 
-      - "8537:8080"
     user: 991:991
     volumes:
       - ./volumes/pictrs:/mnt

docker/dev/nginx.conf

@@ -0,0 +1,51 @@
+worker_processes  1;
+events {
+    worker_connections  1024;
+}
+http {
+    upstream lemmy {
+        server "lemmy:8536";
+    }
+    upstream lemmy-ui {
+        server "lemmy-ui:1234";
+    }
+    server {
+      listen       1236;
+      server_name  localhost;
+
+      # frontend
+      location / {
+        set $proxpass "http://lemmy-ui";
+        if ($http_accept = "application/activity+json") {
+          set $proxpass "http://lemmy";
+        }
+        if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") {
+          set $proxpass "http://lemmy";
+        }
+        if ($request_method = POST) {
+          set $proxpass "http://lemmy";
+        }
+        proxy_pass $proxpass;
+
+        rewrite ^(.+)/+$ $1 permanent;
+
+        # Send actual client IP upstream
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      }
+
+      # backend
+      location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) {
+        proxy_pass "http://lemmy";
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        # Add IP forwarding headers
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      }
+    }
+}

docker/federation/docker-compose.yml

@@ -11,7 +11,7 @@ services:
       - "8580:8580"
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf
-    restart: on-failure
+    restart: always
     depends_on:
       - pictrs
       - lemmy-alpha-ui