mirror of
https://github.com/LemmyNet/lemmy.git
synced 2024-11-21 14:17:08 +00:00
Add logging for pictrs uploads (#3927)
* Add logging for pictrs uploads * cleanup
This commit is contained in:
parent
797d26fdf4
commit
fe3ebea95a
|
@ -1,17 +1,19 @@
|
||||||
use actix_web::web::{Data, Json};
|
use actix_web::web::{Data, Json};
|
||||||
use lemmy_api_common::{
|
use lemmy_api_common::{
|
||||||
context::LemmyContext,
|
context::LemmyContext,
|
||||||
request::purge_image_from_pictrs,
|
request::delete_image_from_pictrs,
|
||||||
site::{PurgeItemResponse, PurgePerson},
|
site::{PurgeItemResponse, PurgePerson},
|
||||||
utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_person, sanitize_html_opt},
|
utils::{is_admin, local_user_view_from_jwt, sanitize_html_opt},
|
||||||
};
|
};
|
||||||
use lemmy_db_schema::{
|
use lemmy_db_schema::{
|
||||||
source::{
|
source::{
|
||||||
|
image_upload::ImageUpload,
|
||||||
moderator::{AdminPurgePerson, AdminPurgePersonForm},
|
moderator::{AdminPurgePerson, AdminPurgePersonForm},
|
||||||
person::Person,
|
person::Person,
|
||||||
},
|
},
|
||||||
traits::Crud,
|
traits::Crud,
|
||||||
};
|
};
|
||||||
|
use lemmy_db_views::structs::LocalUserView;
|
||||||
use lemmy_utils::error::LemmyError;
|
use lemmy_utils::error::LemmyError;
|
||||||
|
|
||||||
#[tracing::instrument(skip(context))]
|
#[tracing::instrument(skip(context))]
|
||||||
|
@ -26,18 +28,17 @@ pub async fn purge_person(
|
||||||
|
|
||||||
// Read the person to get their images
|
// Read the person to get their images
|
||||||
let person_id = data.person_id;
|
let person_id = data.person_id;
|
||||||
let person = Person::read(&mut context.pool(), person_id).await?;
|
|
||||||
|
|
||||||
if let Some(banner) = person.banner {
|
let local_user = LocalUserView::read_person(&mut context.pool(), person_id).await?;
|
||||||
purge_image_from_pictrs(&banner, &context).await.ok();
|
let pictrs_uploads =
|
||||||
|
ImageUpload::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
|
||||||
|
|
||||||
|
for upload in pictrs_uploads {
|
||||||
|
delete_image_from_pictrs(&upload.pictrs_alias, &upload.pictrs_delete_token, &context)
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(avatar) = person.avatar {
|
|
||||||
purge_image_from_pictrs(&avatar, &context).await.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
purge_image_posts_for_person(person_id, &context).await?;
|
|
||||||
|
|
||||||
Person::delete(&mut context.pool(), person_id).await?;
|
Person::delete(&mut context.pool(), person_id).await?;
|
||||||
|
|
||||||
// Mod tables
|
// Mod tables
|
||||||
|
|
|
@ -158,7 +158,6 @@ pub async fn purge_image_from_pictrs(
|
||||||
image_url: &Url,
|
image_url: &Url,
|
||||||
context: &LemmyContext,
|
context: &LemmyContext,
|
||||||
) -> Result<(), LemmyError> {
|
) -> Result<(), LemmyError> {
|
||||||
let pictrs_config = context.settings().pictrs_config()?;
|
|
||||||
is_image_content_type(context.client(), image_url).await?;
|
is_image_content_type(context.client(), image_url).await?;
|
||||||
|
|
||||||
let alias = image_url
|
let alias = image_url
|
||||||
|
@ -167,7 +166,15 @@ pub async fn purge_image_from_pictrs(
|
||||||
.next_back()
|
.next_back()
|
||||||
.ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?;
|
.ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?;
|
||||||
|
|
||||||
let purge_url = format!("{}/internal/purge?alias={}", pictrs_config.url, alias);
|
purge_image_from_pictrs_by_alias(alias, context).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn purge_image_from_pictrs_by_alias(
|
||||||
|
alias: &str,
|
||||||
|
context: &LemmyContext,
|
||||||
|
) -> Result<(), LemmyError> {
|
||||||
|
let pictrs_config = context.settings().pictrs_config()?;
|
||||||
|
let purge_url = format!("{}internal/purge?alias={}", pictrs_config.url, alias);
|
||||||
|
|
||||||
let pictrs_api_key = pictrs_config
|
let pictrs_api_key = pictrs_config
|
||||||
.api_key
|
.api_key
|
||||||
|
@ -189,6 +196,26 @@ pub async fn purge_image_from_pictrs(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn delete_image_from_pictrs(
|
||||||
|
alias: &str,
|
||||||
|
delete_token: &str,
|
||||||
|
context: &LemmyContext,
|
||||||
|
) -> Result<(), LemmyError> {
|
||||||
|
let pictrs_config = context.settings().pictrs_config()?;
|
||||||
|
let url = format!(
|
||||||
|
"{}image/delete/{}/{}",
|
||||||
|
pictrs_config.url, &delete_token, &alias
|
||||||
|
);
|
||||||
|
context
|
||||||
|
.client()
|
||||||
|
.delete(&url)
|
||||||
|
.timeout(REQWEST_TIMEOUT)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(LemmyError::from)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Both are options, since the URL might be either an html page, or an image
|
/// Both are options, since the URL might be either an html page, or an image
|
||||||
/// Returns the SiteMetadata, and an image URL, if there is a picture associated
|
/// Returns the SiteMetadata, and an image URL, if there is a picture associated
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
|
|
47
crates/db_schema/src/impls/image_upload.rs
Normal file
47
crates/db_schema/src/impls/image_upload.rs
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
use crate::{
|
||||||
|
newtypes::{ImageUploadId, LocalUserId},
|
||||||
|
schema::image_upload::dsl::{image_upload, local_user_id, pictrs_alias},
|
||||||
|
source::image_upload::{ImageUpload, ImageUploadForm},
|
||||||
|
utils::{get_conn, DbPool},
|
||||||
|
};
|
||||||
|
use diesel::{insert_into, result::Error, ExpressionMethods, QueryDsl, Table};
|
||||||
|
use diesel_async::RunQueryDsl;
|
||||||
|
|
||||||
|
impl ImageUpload {
|
||||||
|
pub async fn create(pool: &mut DbPool<'_>, form: &ImageUploadForm) -> Result<Self, Error> {
|
||||||
|
let conn = &mut get_conn(pool).await?;
|
||||||
|
insert_into(image_upload)
|
||||||
|
.values(form)
|
||||||
|
.get_result::<Self>(conn)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_all_by_local_user_id(
|
||||||
|
pool: &mut DbPool<'_>,
|
||||||
|
user_id: &LocalUserId,
|
||||||
|
) -> Result<Vec<Self>, Error> {
|
||||||
|
let conn = &mut get_conn(pool).await?;
|
||||||
|
image_upload
|
||||||
|
.filter(local_user_id.eq(user_id))
|
||||||
|
.select(image_upload::all_columns())
|
||||||
|
.load::<ImageUpload>(conn)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete(
|
||||||
|
pool: &mut DbPool<'_>,
|
||||||
|
image_upload_id: ImageUploadId,
|
||||||
|
) -> Result<usize, Error> {
|
||||||
|
let conn = &mut get_conn(pool).await?;
|
||||||
|
diesel::delete(image_upload.find(image_upload_id))
|
||||||
|
.execute(conn)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
|
||||||
|
let conn = &mut get_conn(pool).await?;
|
||||||
|
diesel::delete(image_upload.filter(pictrs_alias.eq(alias)))
|
||||||
|
.execute(conn)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
|
@ -10,6 +10,7 @@ pub mod custom_emoji;
|
||||||
pub mod email_verification;
|
pub mod email_verification;
|
||||||
pub mod federation_allowlist;
|
pub mod federation_allowlist;
|
||||||
pub mod federation_blocklist;
|
pub mod federation_blocklist;
|
||||||
|
pub mod image_upload;
|
||||||
pub mod instance;
|
pub mod instance;
|
||||||
pub mod language;
|
pub mod language;
|
||||||
pub mod local_site;
|
pub mod local_site;
|
||||||
|
|
|
@ -137,6 +137,12 @@ pub struct CommunityLanguageId(pub i32);
|
||||||
/// The comment reply id.
|
/// The comment reply id.
|
||||||
pub struct CommentReplyId(i32);
|
pub struct CommentReplyId(i32);
|
||||||
|
|
||||||
|
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
|
||||||
|
#[cfg_attr(feature = "full", derive(DieselNewType, TS))]
|
||||||
|
#[cfg_attr(feature = "full", ts(export))]
|
||||||
|
/// The Image Upload id.
|
||||||
|
pub struct ImageUploadId(i32);
|
||||||
|
|
||||||
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
|
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
|
||||||
#[cfg_attr(feature = "full", derive(DieselNewType, TS))]
|
#[cfg_attr(feature = "full", derive(DieselNewType, TS))]
|
||||||
#[cfg_attr(feature = "full", ts(export))]
|
#[cfg_attr(feature = "full", ts(export))]
|
||||||
|
|
|
@ -299,6 +299,16 @@ diesel::table! {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
diesel::table! {
|
||||||
|
image_upload (id) {
|
||||||
|
id -> Int4,
|
||||||
|
local_user_id -> Int4,
|
||||||
|
pictrs_alias -> Text,
|
||||||
|
pictrs_delete_token -> Text,
|
||||||
|
published -> Timestamptz,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
diesel::table! {
|
diesel::table! {
|
||||||
instance (id) {
|
instance (id) {
|
||||||
id -> Int4,
|
id -> Int4,
|
||||||
|
@ -405,9 +415,9 @@ diesel::table! {
|
||||||
totp_2fa_secret -> Nullable<Text>,
|
totp_2fa_secret -> Nullable<Text>,
|
||||||
totp_2fa_url -> Nullable<Text>,
|
totp_2fa_url -> Nullable<Text>,
|
||||||
open_links_in_new_tab -> Bool,
|
open_links_in_new_tab -> Bool,
|
||||||
|
infinite_scroll_enabled -> Bool,
|
||||||
blur_nsfw -> Bool,
|
blur_nsfw -> Bool,
|
||||||
auto_expand -> Bool,
|
auto_expand -> Bool,
|
||||||
infinite_scroll_enabled -> Bool,
|
|
||||||
admin -> Bool,
|
admin -> Bool,
|
||||||
post_listing_mode -> PostListingModeEnum,
|
post_listing_mode -> PostListingModeEnum,
|
||||||
}
|
}
|
||||||
|
@ -893,6 +903,7 @@ diesel::joinable!(custom_emoji_keyword -> custom_emoji (custom_emoji_id));
|
||||||
diesel::joinable!(email_verification -> local_user (local_user_id));
|
diesel::joinable!(email_verification -> local_user (local_user_id));
|
||||||
diesel::joinable!(federation_allowlist -> instance (instance_id));
|
diesel::joinable!(federation_allowlist -> instance (instance_id));
|
||||||
diesel::joinable!(federation_blocklist -> instance (instance_id));
|
diesel::joinable!(federation_blocklist -> instance (instance_id));
|
||||||
|
diesel::joinable!(image_upload -> local_user (local_user_id));
|
||||||
diesel::joinable!(local_site -> site (site_id));
|
diesel::joinable!(local_site -> site (site_id));
|
||||||
diesel::joinable!(local_site_rate_limit -> local_site (local_site_id));
|
diesel::joinable!(local_site_rate_limit -> local_site (local_site_id));
|
||||||
diesel::joinable!(local_user -> person (person_id));
|
diesel::joinable!(local_user -> person (person_id));
|
||||||
|
@ -967,6 +978,7 @@ diesel::allow_tables_to_appear_in_same_query!(
|
||||||
email_verification,
|
email_verification,
|
||||||
federation_allowlist,
|
federation_allowlist,
|
||||||
federation_blocklist,
|
federation_blocklist,
|
||||||
|
image_upload,
|
||||||
instance,
|
instance,
|
||||||
language,
|
language,
|
||||||
local_site,
|
local_site,
|
||||||
|
|
36
crates/db_schema/src/source/image_upload.rs
Normal file
36
crates/db_schema/src/source/image_upload.rs
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
use crate::newtypes::{ImageUploadId, LocalUserId};
|
||||||
|
#[cfg(feature = "full")]
|
||||||
|
use crate::schema::image_upload;
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_with::skip_serializing_none;
|
||||||
|
use std::fmt::Debug;
|
||||||
|
#[cfg(feature = "full")]
|
||||||
|
use ts_rs::TS;
|
||||||
|
use typed_builder::TypedBuilder;
|
||||||
|
|
||||||
|
#[skip_serializing_none]
|
||||||
|
#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))]
|
||||||
|
#[cfg_attr(feature = "full", diesel(table_name = image_upload))]
|
||||||
|
#[cfg_attr(feature = "full", ts(export))]
|
||||||
|
#[cfg_attr(
|
||||||
|
feature = "full",
|
||||||
|
diesel(belongs_to(crate::source::local_user::LocalUser))
|
||||||
|
)]
|
||||||
|
pub struct ImageUpload {
|
||||||
|
pub id: ImageUploadId,
|
||||||
|
pub local_user_id: LocalUserId,
|
||||||
|
pub pictrs_alias: String,
|
||||||
|
pub pictrs_delete_token: String,
|
||||||
|
pub published: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, TypedBuilder)]
|
||||||
|
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
|
||||||
|
#[cfg_attr(feature = "full", diesel(table_name = image_upload))]
|
||||||
|
pub struct ImageUploadForm {
|
||||||
|
pub local_user_id: LocalUserId,
|
||||||
|
pub pictrs_alias: String,
|
||||||
|
pub pictrs_delete_token: String,
|
||||||
|
}
|
|
@ -15,6 +15,7 @@ pub mod custom_emoji_keyword;
|
||||||
pub mod email_verification;
|
pub mod email_verification;
|
||||||
pub mod federation_allowlist;
|
pub mod federation_allowlist;
|
||||||
pub mod federation_blocklist;
|
pub mod federation_blocklist;
|
||||||
|
pub mod image_upload;
|
||||||
pub mod instance;
|
pub mod instance;
|
||||||
pub mod language;
|
pub mod language;
|
||||||
pub mod local_site;
|
pub mod local_site;
|
||||||
|
|
|
@ -12,7 +12,13 @@ use actix_web::{
|
||||||
};
|
};
|
||||||
use futures::stream::{Stream, StreamExt};
|
use futures::stream::{Stream, StreamExt};
|
||||||
use lemmy_api_common::{context::LemmyContext, utils::local_user_view_from_jwt};
|
use lemmy_api_common::{context::LemmyContext, utils::local_user_view_from_jwt};
|
||||||
use lemmy_db_schema::source::local_site::LocalSite;
|
use lemmy_db_schema::{
|
||||||
|
newtypes::LocalUserId,
|
||||||
|
source::{
|
||||||
|
image_upload::{ImageUpload, ImageUploadForm},
|
||||||
|
local_site::LocalSite,
|
||||||
|
},
|
||||||
|
};
|
||||||
use lemmy_utils::{claims::Claims, rate_limit::RateLimitCell, REQWEST_TIMEOUT};
|
use lemmy_utils::{claims::Claims, rate_limit::RateLimitCell, REQWEST_TIMEOUT};
|
||||||
use reqwest::Body;
|
use reqwest::Body;
|
||||||
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
|
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
|
||||||
|
@ -95,8 +101,8 @@ async fn upload(
|
||||||
let jwt = req.cookie("jwt").ok_or(error::ErrorUnauthorized(
|
let jwt = req.cookie("jwt").ok_or(error::ErrorUnauthorized(
|
||||||
"No auth header for picture upload",
|
"No auth header for picture upload",
|
||||||
))?;
|
))?;
|
||||||
|
let claims = Claims::decode(jwt.value(), &context.secret().jwt_secret);
|
||||||
if Claims::decode(jwt.value(), &context.secret().jwt_secret).is_err() {
|
if claims.is_err() {
|
||||||
return Ok(HttpResponse::Unauthorized().finish());
|
return Ok(HttpResponse::Unauthorized().finish());
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -108,7 +114,6 @@ async fn upload(
|
||||||
if let Some(addr) = req.head().peer_addr {
|
if let Some(addr) = req.head().peer_addr {
|
||||||
client_req = client_req.header("X-Forwarded-For", addr.to_string())
|
client_req = client_req.header("X-Forwarded-For", addr.to_string())
|
||||||
};
|
};
|
||||||
|
|
||||||
let res = client_req
|
let res = client_req
|
||||||
.body(Body::wrap_stream(make_send(body)))
|
.body(Body::wrap_stream(make_send(body)))
|
||||||
.send()
|
.send()
|
||||||
|
@ -117,6 +122,19 @@ async fn upload(
|
||||||
|
|
||||||
let status = res.status();
|
let status = res.status();
|
||||||
let images = res.json::<Images>().await.map_err(error::ErrorBadRequest)?;
|
let images = res.json::<Images>().await.map_err(error::ErrorBadRequest)?;
|
||||||
|
if let Some(images) = &images.files {
|
||||||
|
let local_user_id = LocalUserId(claims?.claims.sub);
|
||||||
|
for uploaded_image in images {
|
||||||
|
let form = ImageUploadForm {
|
||||||
|
local_user_id,
|
||||||
|
pictrs_alias: uploaded_image.file.to_string(),
|
||||||
|
pictrs_delete_token: uploaded_image.delete_token.to_string(),
|
||||||
|
};
|
||||||
|
ImageUpload::create(&mut context.pool(), &form)
|
||||||
|
.await
|
||||||
|
.map_err(error::ErrorBadRequest)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok(HttpResponse::build(status).json(images))
|
Ok(HttpResponse::build(status).json(images))
|
||||||
}
|
}
|
||||||
|
@ -215,6 +233,10 @@ async fn delete(
|
||||||
|
|
||||||
let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
|
let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
|
||||||
|
|
||||||
|
ImageUpload::delete_by_alias(&mut context.pool(), &file)
|
||||||
|
.await
|
||||||
|
.map_err(error::ErrorBadRequest)?;
|
||||||
|
|
||||||
Ok(HttpResponse::build(res.status()).body(BodyStream::new(res.bytes_stream())))
|
Ok(HttpResponse::build(res.status()).body(BodyStream::new(res.bytes_stream())))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
2
migrations/2023-08-31-205559_add_image_upload/down.sql
Normal file
2
migrations/2023-08-31-205559_add_image_upload/down.sql
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
DROP TABLE image_upload;
|
||||||
|
|
10
migrations/2023-08-31-205559_add_image_upload/up.sql
Normal file
10
migrations/2023-08-31-205559_add_image_upload/up.sql
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
CREATE TABLE image_upload (
|
||||||
|
id serial PRIMARY KEY,
|
||||||
|
local_user_id int REFERENCES local_user ON UPDATE CASCADE ON DELETE CASCADE NOT NULL,
|
||||||
|
pictrs_alias text NOT NULL UNIQUE,
|
||||||
|
pictrs_delete_token text NOT NULL,
|
||||||
|
published timestamptz DEFAULT now() NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_image_upload_local_user_id ON image_upload (local_user_id);
|
||||||
|
|
|
@ -11,5 +11,5 @@ find migrations -type f -name "*.sql" -print0 | while read -d $'\0' FILE
|
||||||
do
|
do
|
||||||
TMP_FILE="/tmp/tmp_pg_format.sql"
|
TMP_FILE="/tmp/tmp_pg_format.sql"
|
||||||
pg_format $FILE > $TMP_FILE
|
pg_format $FILE > $TMP_FILE
|
||||||
diff $FILE $TMP_FILE
|
diff -u $FILE $TMP_FILE
|
||||||
done
|
done
|
||||||
|
|
Loading…
Reference in a new issue