Avoid using proxy for pictrs requests (fixes #3489) (#4072)

* Avoid using proxy for pictrs requests (fixes #3489)

* fmt
This commit is contained in:
Nutomic 2023-10-20 17:09:34 +02:00 committed by GitHub
parent dfc74835b1
commit ec0a707110
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 33 additions and 53 deletions

View file

@ -8,6 +8,7 @@ use lemmy_utils::{
REQWEST_TIMEOUT,
};
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::{Client, ClientBuilder};
use reqwest_middleware::ClientWithMiddleware;
use serde::Deserialize;
use tracing::info;
@ -288,12 +289,17 @@ async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Resu
}
}
pub fn build_user_agent(settings: &Settings) -> String {
format!(
pub fn client_builder(settings: &Settings) -> ClientBuilder {
let user_agent = format!(
"Lemmy/{}; +{}",
VERSION,
settings.get_protocol_and_hostname()
)
);
Client::builder()
.user_agent(user_agent.clone())
.timeout(REQWEST_TIMEOUT)
.connect_timeout(REQWEST_TIMEOUT)
}
#[cfg(test)]
@ -301,12 +307,7 @@ mod tests {
#![allow(clippy::unwrap_used)]
#![allow(clippy::indexing_slicing)]
use crate::request::{
build_user_agent,
fetch_site_metadata,
html_to_site_metadata,
SiteMetadata,
};
use crate::request::{client_builder, fetch_site_metadata, html_to_site_metadata, SiteMetadata};
use lemmy_utils::settings::SETTINGS;
use url::Url;
@ -314,11 +315,7 @@ mod tests {
#[tokio::test]
async fn test_site_metadata() {
let settings = &SETTINGS.clone();
let client = reqwest::Client::builder()
.user_agent(build_user_agent(settings))
.build()
.unwrap()
.into();
let client = client_builder(settings).build().unwrap().into();
let sample_url = Url::parse("https://gitlab.com/IzzyOnDroid/repo/-/wikis/FAQ").unwrap();
let sample_res = fetch_site_metadata(&client, &sample_url).await.unwrap();
assert_eq!(

View file

@ -59,10 +59,10 @@ pub(crate) mod tests {
use activitypub_federation::config::{Data, FederationConfig};
use anyhow::anyhow;
use lemmy_api_common::{context::LemmyContext, request::build_user_agent};
use lemmy_api_common::{context::LemmyContext, request::client_builder};
use lemmy_db_schema::{source::secret::Secret, utils::build_db_pool_for_tests};
use lemmy_utils::{rate_limit::RateLimitCell, settings::SETTINGS};
use reqwest::{Client, Request, Response};
use reqwest::{Request, Response};
use reqwest_middleware::{ClientBuilder, Middleware, Next};
use task_local_extensions::Extensions;
@ -86,11 +86,7 @@ pub(crate) mod tests {
// call this to run migrations
let pool = build_db_pool_for_tests().await;
let settings = SETTINGS.clone();
let client = Client::builder()
.user_agent(build_user_agent(&settings))
.build()
.unwrap();
let client = client_builder(&SETTINGS).build().unwrap();
let client = ClientBuilder::new(client).with(BlockedMiddleware).build();
let secret = Secret {

View file

@ -55,7 +55,7 @@ services:
lemmy-ui:
# use "image" to pull down an already compiled lemmy-ui. make sure to comment out "build".
image: dessalines/lemmy-ui:0.18.4
image: dessalines/lemmy-ui:0.19.0-rc.3
# platform: linux/x86_64 # no arm64 support. uncomment platform if using m1.
# use "build" to build your local lemmy ui image for development. make sure to comment out "image".
# run: docker compose up --build

View file

@ -45,9 +45,9 @@ if [ "$ARCH" = 'arm64' ]; then
fi
echo "$LOG_PREFIX Initializing images in the background. Please be patient if compiling from source..."
docker compose up -d --build
docker compose up --build
else
sudo docker compose up -d --build
sudo docker compose up --build
fi
echo "$LOG_PREFIX Complete! You can now access the UI at http://localhost:1236."

View file

@ -28,7 +28,7 @@ use clap::{ArgAction, Parser};
use lemmy_api_common::{
context::LemmyContext,
lemmy_db_views::structs::SiteView,
request::build_user_agent,
request::client_builder,
send_activity::{ActivityChannel, MATCH_OUTGOING_ACTIVITIES},
utils::{
check_private_instance_and_federation_enabled,
@ -52,11 +52,10 @@ use lemmy_utils::{
response::jsonify_plain_text_errors,
settings::{structs::Settings, SETTINGS},
};
use reqwest::Client;
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_tracing::TracingMiddleware;
use serde_json::json;
use std::{env, ops::Deref, time::Duration};
use std::{env, ops::Deref};
use tokio::signal::unix::SignalKind;
use tracing::subscriber::set_global_default;
use tracing_actix_web::TracingLogger;
@ -112,13 +111,9 @@ pub struct CmdArgs {
#[arg(long, default_value_t = 1)]
federate_process_count: i32,
}
/// Max timeout for http requests
pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
/// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
let settings = SETTINGS.to_owned();
// return error 503 while running db migrations and startup tasks
let mut startup_server_handle = None;
if args.http_server {
@ -126,14 +121,14 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
}
// Run the DB migrations
let db_url = get_database_url(Some(&settings));
let db_url = get_database_url(Some(&SETTINGS));
run_migrations(&db_url);
// Set up the connection pool
let pool = build_db_pool(&settings).await?;
let pool = build_db_pool(&SETTINGS).await?;
// Run the Code-required migrations
run_advanced_migrations(&mut (&pool).into(), &settings).await?;
run_advanced_migrations(&mut (&pool).into(), &SETTINGS).await?;
// Initialize the secrets
let secret = Secret::init(&mut (&pool).into())
@ -148,7 +143,7 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
let federation_enabled = local_site.federation_enabled;
if federation_enabled {
println!("federation enabled, host is {}", &settings.hostname);
println!("federation enabled, host is {}", &SETTINGS.hostname);
}
check_private_instance_and_federation_enabled(&local_site)?;
@ -160,25 +155,12 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
println!(
"Starting http server at {}:{}",
settings.bind, settings.port
SETTINGS.bind, SETTINGS.port
);
let user_agent = build_user_agent(&settings);
let reqwest_client = Client::builder()
.user_agent(user_agent.clone())
.timeout(REQWEST_TIMEOUT)
.connect_timeout(REQWEST_TIMEOUT)
.build()?;
let client = ClientBuilder::new(reqwest_client.clone())
let client = ClientBuilder::new(client_builder(&SETTINGS).build()?)
.with(TracingMiddleware::default())
.build();
// Pictrs cannot use the retry middleware
let pictrs_client = ClientBuilder::new(reqwest_client.clone())
.with(TracingMiddleware::default())
.build();
let context = LemmyContext::create(
pool.clone(),
client.clone(),
@ -192,10 +174,10 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
}
#[cfg(feature = "prometheus-metrics")]
serve_prometheus(settings.prometheus.as_ref(), context.clone());
serve_prometheus(SETTINGS.prometheus.as_ref(), context.clone());
let federation_config = FederationConfig::builder()
.domain(settings.hostname.clone())
.domain(SETTINGS.hostname.clone())
.app_data(context.clone())
.client(client.clone())
.http_fetch_limit(FEDERATION_HTTP_FETCH_LIMIT)
@ -217,9 +199,14 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
if let Some(startup_server_handle) = startup_server_handle {
startup_server_handle.stop(true).await;
}
// Pictrs cannot use proxy
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
.with(TracingMiddleware::default())
.build();
Some(create_http_server(
federation_config.clone(),
settings.clone(),
SETTINGS.clone(),
federation_enabled,
pictrs_client,
)?)