mirror of
https://github.com/LemmyNet/lemmy.git
synced 2024-11-08 09:24:17 +00:00
Rewrite images to use local proxy (#4035)
* Add markdown rule to add rel=nofollow for all links * Add markdown image rule to add local image proxy (fixes #1036) * comments * rewrite markdown image links working * add comment * perform markdown image processing in api/apub receivers * clippy * add db table to validate proxied links * rewrite link fields for avatar, banner etc * sql fmt * proxy links received over federation * add config option * undo post.url rewriting, move http route definition * add tests * proxy images through pictrs * testing * cleanup request.rs file * more cleanup (fixes #2611) * include url content type when sending post over apub (fixes #2611) * store post url content type in db * should be media_type * get rid of cache_remote_thumbnails setting, instead automatically take thumbnail from federation data if available. * fix tests * add setting disable_external_link_previews * federate post url as image depending on mime type * change setting again * machete * invert * support custom emoji * clippy * update defaults * add image proxy test, fix test * fix test * clippy * revert accidental changes * address review * clippy * Markdown link rule-dess (#4356) * Extracting opengraph_data to its own type. * A few additions for markdown-link-rule. --------- Co-authored-by: Nutomic <me@nutomic.com> * fix setting * use enum for image proxy setting * fix test configs * add config backwards compat * clippy * machete --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
This commit is contained in:
parent
1782aafd10
commit
e8a52d3a5c
11
Cargo.lock
generated
11
Cargo.lock
generated
|
@ -2571,6 +2571,8 @@ version = "0.19.3"
|
|||
dependencies = [
|
||||
"activitypub_federation",
|
||||
"actix-web",
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"encoding",
|
||||
"enum-map",
|
||||
|
@ -2582,8 +2584,8 @@ dependencies = [
|
|||
"lemmy_db_views_actor",
|
||||
"lemmy_db_views_moderator",
|
||||
"lemmy_utils",
|
||||
"mime",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pretty_assertions",
|
||||
"regex",
|
||||
"reqwest",
|
||||
|
@ -2592,10 +2594,12 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_with",
|
||||
"serial_test",
|
||||
"task-local-extensions",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"ts-rs",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"webpage",
|
||||
]
|
||||
|
@ -2648,14 +2652,12 @@ dependencies = [
|
|||
"once_cell",
|
||||
"pretty_assertions",
|
||||
"reqwest",
|
||||
"reqwest-middleware",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"serial_test",
|
||||
"stringreader",
|
||||
"strum_macros",
|
||||
"task-local-extensions",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
|
@ -2811,6 +2813,7 @@ dependencies = [
|
|||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
@ -2874,7 +2877,6 @@ dependencies = [
|
|||
"markdown-it",
|
||||
"once_cell",
|
||||
"openssl",
|
||||
"percent-encoding",
|
||||
"pretty_assertions",
|
||||
"regex",
|
||||
"reqwest",
|
||||
|
@ -2891,6 +2893,7 @@ dependencies = [
|
|||
"tracing-error",
|
||||
"ts-rs",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
|
|
|
@ -146,7 +146,6 @@ strum_macros = "0.25.3"
|
|||
itertools = "0.12.0"
|
||||
futures = "0.3.30"
|
||||
http = "0.2.11"
|
||||
percent-encoding = "2.3.1"
|
||||
rosetta-i18n = "0.1.3"
|
||||
opentelemetry = { version = "0.19.0", features = ["rt-tokio"] }
|
||||
tracing-opentelemetry = { version = "0.19.0" }
|
||||
|
@ -155,6 +154,7 @@ rustls = { version = "0.21.10", features = ["dangerous_configuration"] }
|
|||
futures-util = "0.3.30"
|
||||
tokio-postgres = "0.7.10"
|
||||
tokio-postgres-rustls = "0.10.0"
|
||||
urlencoding = "2.1.3"
|
||||
enum-map = "2.7"
|
||||
moka = { version = "0.12.4", features = ["future"] }
|
||||
i-love-jesus = { version = "0.1.0" }
|
||||
|
|
|
@ -7,16 +7,23 @@ import {
|
|||
PurgePost,
|
||||
} from "lemmy-js-client";
|
||||
import {
|
||||
alpha,
|
||||
alphaImage,
|
||||
alphaUrl,
|
||||
beta,
|
||||
betaUrl,
|
||||
createCommunity,
|
||||
createPost,
|
||||
delta,
|
||||
epsilon,
|
||||
gamma,
|
||||
getSite,
|
||||
registerUser,
|
||||
resolveBetaCommunity,
|
||||
resolvePost,
|
||||
setupLogins,
|
||||
unfollowRemotes,
|
||||
waitForPost,
|
||||
} from "./shared";
|
||||
const downloadFileSync = require("download-file-sync");
|
||||
|
||||
|
@ -29,9 +36,8 @@ afterAll(() => {
|
|||
test("Upload image and delete it", async () => {
|
||||
// Upload test image. We use a simple string buffer as pictrs doesnt require an actual image
|
||||
// in testing mode.
|
||||
const upload_image = Buffer.from("test");
|
||||
const upload_form: UploadImage = {
|
||||
image: upload_image,
|
||||
image: Buffer.from("test"),
|
||||
};
|
||||
const upload = await alphaImage.uploadImage(upload_form);
|
||||
expect(upload.files![0].file).toBeDefined();
|
||||
|
@ -60,9 +66,8 @@ test("Purge user, uploaded image removed", async () => {
|
|||
let user = await registerUser(alphaImage, alphaUrl);
|
||||
|
||||
// upload test image
|
||||
const upload_image = Buffer.from("test");
|
||||
const upload_form: UploadImage = {
|
||||
image: upload_image,
|
||||
image: Buffer.from("test"),
|
||||
};
|
||||
const upload = await user.uploadImage(upload_form);
|
||||
expect(upload.files![0].file).toBeDefined();
|
||||
|
@ -91,9 +96,8 @@ test("Purge post, linked image removed", async () => {
|
|||
let user = await registerUser(beta, betaUrl);
|
||||
|
||||
// upload test image
|
||||
const upload_image = Buffer.from("test");
|
||||
const upload_form: UploadImage = {
|
||||
image: upload_image,
|
||||
image: Buffer.from("test"),
|
||||
};
|
||||
const upload = await user.uploadImage(upload_form);
|
||||
expect(upload.files![0].file).toBeDefined();
|
||||
|
@ -124,3 +128,79 @@ test("Purge post, linked image removed", async () => {
|
|||
const content2 = downloadFileSync(upload.url);
|
||||
expect(content2).toBe("");
|
||||
});
|
||||
|
||||
test("Images in remote post are proxied if setting enabled", async () => {
|
||||
let user = await registerUser(beta, betaUrl);
|
||||
let community = await createCommunity(gamma);
|
||||
|
||||
const upload_form: UploadImage = {
|
||||
image: Buffer.from("test"),
|
||||
};
|
||||
const upload = await user.uploadImage(upload_form);
|
||||
let post = await createPost(
|
||||
gamma,
|
||||
community.community_view.community.id,
|
||||
upload.url,
|
||||
"![](http://example.com/image2.png)",
|
||||
);
|
||||
expect(post.post_view.post).toBeDefined();
|
||||
|
||||
// remote image gets proxied after upload
|
||||
expect(
|
||||
post.post_view.post.url?.startsWith(
|
||||
"http://lemmy-gamma:8561/api/v3/image_proxy?url",
|
||||
),
|
||||
).toBeTruthy();
|
||||
expect(
|
||||
post.post_view.post.body?.startsWith(
|
||||
"![](http://lemmy-gamma:8561/api/v3/image_proxy?url",
|
||||
),
|
||||
).toBeTruthy();
|
||||
|
||||
let epsilonPost = await resolvePost(epsilon, post.post_view.post);
|
||||
expect(epsilonPost.post).toBeDefined();
|
||||
|
||||
// remote image gets proxied after federation
|
||||
expect(
|
||||
epsilonPost.post!.post.url?.startsWith(
|
||||
"http://lemmy-epsilon:8581/api/v3/image_proxy?url",
|
||||
),
|
||||
).toBeTruthy();
|
||||
expect(
|
||||
epsilonPost.post!.post.body?.startsWith(
|
||||
"![](http://lemmy-epsilon:8581/api/v3/image_proxy?url",
|
||||
),
|
||||
).toBeTruthy();
|
||||
});
|
||||
|
||||
test("No image proxying if setting is disabled", async () => {
|
||||
let user = await registerUser(beta, betaUrl);
|
||||
let community = await createCommunity(alpha);
|
||||
|
||||
const upload_form: UploadImage = {
|
||||
image: Buffer.from("test"),
|
||||
};
|
||||
const upload = await user.uploadImage(upload_form);
|
||||
let post = await createPost(
|
||||
alpha,
|
||||
community.community_view.community.id,
|
||||
upload.url,
|
||||
"![](http://example.com/image2.png)",
|
||||
);
|
||||
expect(post.post_view.post).toBeDefined();
|
||||
|
||||
// remote image doesnt get proxied after upload
|
||||
expect(
|
||||
post.post_view.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"),
|
||||
).toBeTruthy();
|
||||
expect(post.post_view.post.body).toBe("![](http://example.com/image2.png)");
|
||||
|
||||
let gammaPost = await resolvePost(delta, post.post_view.post);
|
||||
expect(gammaPost.post).toBeDefined();
|
||||
|
||||
// remote image doesnt get proxied after federation
|
||||
expect(
|
||||
gammaPost.post!.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"),
|
||||
).toBeTruthy();
|
||||
expect(gammaPost.post!.post.body).toBe("![](http://example.com/image2.png)");
|
||||
});
|
||||
|
|
|
@ -39,7 +39,7 @@ import {
|
|||
loginUser,
|
||||
} from "./shared";
|
||||
import { PostView } from "lemmy-js-client/dist/types/PostView";
|
||||
import { ResolveObject } from "lemmy-js-client";
|
||||
import { EditSite, ResolveObject } from "lemmy-js-client";
|
||||
|
||||
let betaCommunity: CommunityView | undefined;
|
||||
|
||||
|
@ -72,6 +72,16 @@ function assertPostFederation(postOne?: PostView, postTwo?: PostView) {
|
|||
}
|
||||
|
||||
test("Create a post", async () => {
|
||||
// Setup some allowlists and blocklists
|
||||
let editSiteForm: EditSite = {
|
||||
allowed_instances: ["lemmy-beta"],
|
||||
};
|
||||
await delta.editSite(editSiteForm);
|
||||
|
||||
editSiteForm.allowed_instances = [];
|
||||
editSiteForm.blocked_instances = ["lemmy-alpha"];
|
||||
await epsilon.editSite(editSiteForm);
|
||||
|
||||
if (!betaCommunity) {
|
||||
throw "Missing beta community";
|
||||
}
|
||||
|
@ -109,6 +119,12 @@ test("Create a post", async () => {
|
|||
await expect(
|
||||
resolvePost(epsilon, postRes.post_view.post),
|
||||
).rejects.toStrictEqual(Error("couldnt_find_object"));
|
||||
|
||||
// remove added allow/blocklists
|
||||
editSiteForm.allowed_instances = [];
|
||||
editSiteForm.blocked_instances = [];
|
||||
await delta.editSite(editSiteForm);
|
||||
await epsilon.editSite(editSiteForm);
|
||||
});
|
||||
|
||||
test("Create a post in a non-existent community", async () => {
|
||||
|
|
|
@ -177,13 +177,6 @@ export async function setupLogins() {
|
|||
];
|
||||
await gamma.editSite(editSiteForm);
|
||||
|
||||
editSiteForm.allowed_instances = ["lemmy-beta"];
|
||||
await delta.editSite(editSiteForm);
|
||||
|
||||
editSiteForm.allowed_instances = [];
|
||||
editSiteForm.blocked_instances = ["lemmy-alpha"];
|
||||
await epsilon.editSite(editSiteForm);
|
||||
|
||||
// Create the main alpha/beta communities
|
||||
// Ignore thrown errors of duplicates
|
||||
try {
|
||||
|
@ -203,10 +196,10 @@ export async function createPost(
|
|||
api: LemmyHttp,
|
||||
community_id: number,
|
||||
url: string = "https://example.com/",
|
||||
body = randomString(10),
|
||||
// use example.com for consistent title and embed description
|
||||
name: string = randomString(5),
|
||||
): Promise<PostResponse> {
|
||||
let body = randomString(10);
|
||||
let form: CreatePost = {
|
||||
name,
|
||||
url,
|
||||
|
@ -528,7 +521,7 @@ export async function likeComment(
|
|||
|
||||
export async function createCommunity(
|
||||
api: LemmyHttp,
|
||||
name_: string = randomString(5),
|
||||
name_: string = randomString(10),
|
||||
): Promise<CommunityResponse> {
|
||||
let description = "a sample description";
|
||||
let form: CreateCommunity = {
|
||||
|
|
|
@ -36,22 +36,41 @@
|
|||
# Maximum number of active sql connections
|
||||
pool_size: 30
|
||||
}
|
||||
# Settings related to activitypub federation
|
||||
# Pictrs image server configuration.
|
||||
pictrs: {
|
||||
# Address where pictrs is available (for image hosting)
|
||||
url: "http://localhost:8080/"
|
||||
# Set a custom pictrs API key. ( Required for deleting images )
|
||||
api_key: "string"
|
||||
# By default the thumbnails for external links are stored in pict-rs. This ensures that they
|
||||
# can be reliably retrieved and can be resized using pict-rs APIs. However it also increases
|
||||
# storage usage. In case this is disabled, the Opengraph image is directly returned as
|
||||
# thumbnail.
|
||||
# Backwards compatibility with 0.18.1. False is equivalent to `image_mode: None`, true is
|
||||
# equivalent to `image_mode: StoreLinkPreviews`.
|
||||
#
|
||||
# In some countries it is forbidden to copy preview images from newspaper articles and only
|
||||
# hotlinking is allowed. If that is the case for your instance, make sure that this setting is
|
||||
# disabled.
|
||||
# To be removed in 0.20
|
||||
cache_external_link_previews: true
|
||||
# Specifies how to handle remote images, so that users don't have to connect directly to remote servers.
|
||||
image_mode:
|
||||
# Leave images unchanged, don't generate any local thumbnails for post urls. Instead the the
|
||||
# Opengraph image is directly returned as thumbnail
|
||||
"None"
|
||||
|
||||
# or
|
||||
|
||||
# Generate thumbnails for external post urls and store them persistently in pict-rs. This
|
||||
# ensures that they can be reliably retrieved and can be resized using pict-rs APIs. However
|
||||
# it also increases storage usage.
|
||||
#
|
||||
# This is the default behaviour, and also matches Lemmy 0.18.
|
||||
"StoreLinkPreviews"
|
||||
|
||||
# or
|
||||
|
||||
# If enabled, all images from remote domains are rewritten to pass through `/api/v3/image_proxy`,
|
||||
# including embedded images in markdown. Images are stored temporarily in pict-rs for caching.
|
||||
# This improves privacy as users don't expose their IP to untrusted servers, and decreases load
|
||||
# on other servers. However it increases bandwidth use for the local server.
|
||||
#
|
||||
# Requires pict-rs 0.5
|
||||
"ProxyAllImages"
|
||||
# Timeout for uploading images to pictrs (in seconds)
|
||||
upload_timeout: 30
|
||||
}
|
||||
|
|
|
@ -2,7 +2,12 @@ use actix_web::web::{Data, Json};
|
|||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
person::SaveUserSettings,
|
||||
utils::send_verification_email,
|
||||
utils::{
|
||||
local_site_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_api,
|
||||
send_verification_email,
|
||||
},
|
||||
SuccessResponse,
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
|
@ -12,7 +17,7 @@ use lemmy_db_schema::{
|
|||
person::{Person, PersonUpdateForm},
|
||||
},
|
||||
traits::Crud,
|
||||
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url},
|
||||
utils::diesel_option_overwrite,
|
||||
};
|
||||
use lemmy_db_views::structs::{LocalUserView, SiteView};
|
||||
use lemmy_utils::{
|
||||
|
@ -28,9 +33,11 @@ pub async fn save_user_settings(
|
|||
) -> Result<Json<SuccessResponse>, LemmyError> {
|
||||
let site_view = SiteView::read_local(&mut context.pool()).await?;
|
||||
|
||||
let avatar = diesel_option_overwrite_to_url(&data.avatar)?;
|
||||
let banner = diesel_option_overwrite_to_url(&data.banner)?;
|
||||
let bio = diesel_option_overwrite(data.bio.clone());
|
||||
let slur_regex = local_site_to_slur_regex(&site_view.local_site);
|
||||
let bio = diesel_option_overwrite(process_markdown_opt(&data.bio, &slur_regex, &context).await?);
|
||||
|
||||
let avatar = proxy_image_link_opt_api(&data.avatar, &context).await?;
|
||||
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
|
||||
let display_name = diesel_option_overwrite(data.display_name.clone());
|
||||
let matrix_user_id = diesel_option_overwrite(data.matrix_user_id.clone());
|
||||
let email_deref = data.email.as_deref().map(str::to_lowercase);
|
||||
|
|
|
@ -2,7 +2,7 @@ use actix_web::web::{Data, Json, Query};
|
|||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
post::{GetSiteMetadata, GetSiteMetadataResponse},
|
||||
request::fetch_site_metadata,
|
||||
request::fetch_link_metadata,
|
||||
};
|
||||
use lemmy_utils::error::LemmyError;
|
||||
|
||||
|
@ -11,7 +11,7 @@ pub async fn get_link_metadata(
|
|||
data: Query<GetSiteMetadata>,
|
||||
context: Data<LemmyContext>,
|
||||
) -> Result<Json<GetSiteMetadataResponse>, LemmyError> {
|
||||
let metadata = fetch_site_metadata(context.client(), &data.url).await?;
|
||||
let metadata = fetch_link_metadata(&data.url, false, &context).await?;
|
||||
|
||||
Ok(Json(GetSiteMetadataResponse { metadata }))
|
||||
}
|
||||
|
|
|
@ -8,7 +8,7 @@ use lemmy_api_common::{
|
|||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
image_upload::ImageUpload,
|
||||
images::LocalImage,
|
||||
moderator::{AdminPurgePerson, AdminPurgePersonForm},
|
||||
person::{Person, PersonUpdateForm},
|
||||
},
|
||||
|
@ -31,7 +31,7 @@ pub async fn purge_person(
|
|||
|
||||
if let Ok(local_user) = LocalUserView::read_person(&mut context.pool(), person_id).await {
|
||||
let pictrs_uploads =
|
||||
ImageUpload::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
|
||||
LocalImage::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
|
||||
|
||||
for upload in pictrs_uploads {
|
||||
delete_image_from_pictrs(&upload.pictrs_alias, &upload.pictrs_delete_token, &context)
|
||||
|
|
|
@ -25,7 +25,6 @@ full = [
|
|||
"lemmy_db_views_actor/full",
|
||||
"lemmy_db_views_moderator/full",
|
||||
"activitypub_federation",
|
||||
"percent-encoding",
|
||||
"encoding",
|
||||
"reqwest-middleware",
|
||||
"webpage",
|
||||
|
@ -37,6 +36,7 @@ full = [
|
|||
"futures",
|
||||
"once_cell",
|
||||
"jsonwebtoken",
|
||||
"mime",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
|
@ -54,11 +54,7 @@ tracing = { workspace = true, optional = true }
|
|||
reqwest-middleware = { workspace = true, optional = true }
|
||||
regex = { workspace = true }
|
||||
rosetta-i18n = { workspace = true, optional = true }
|
||||
percent-encoding = { workspace = true, optional = true }
|
||||
webpage = { version = "1.6", default-features = false, features = [
|
||||
"serde",
|
||||
], optional = true }
|
||||
encoding = { version = "0.2.33", optional = true }
|
||||
anyhow = { workspace = true }
|
||||
futures = { workspace = true, optional = true }
|
||||
uuid = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, optional = true }
|
||||
|
@ -66,10 +62,18 @@ reqwest = { workspace = true, optional = true }
|
|||
ts-rs = { workspace = true, optional = true }
|
||||
once_cell = { workspace = true, optional = true }
|
||||
actix-web = { workspace = true, optional = true }
|
||||
enum-map = { workspace = true }
|
||||
urlencoding = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
mime = { version = "0.3.17", optional = true }
|
||||
webpage = { version = "1.6", default-features = false, features = [
|
||||
"serde",
|
||||
], optional = true }
|
||||
encoding = { version = "0.2.33", optional = true }
|
||||
jsonwebtoken = { version = "8.3.0", optional = true }
|
||||
# necessary for wasmt compilation
|
||||
getrandom = { version = "0.2.12", features = ["js"] }
|
||||
enum-map = { workspace = true }
|
||||
task-local-extensions = "0.1.4"
|
||||
|
||||
[package.metadata.cargo-machete]
|
||||
ignored = ["getrandom"]
|
||||
|
|
|
@ -1,13 +1,18 @@
|
|||
use crate::request::client_builder;
|
||||
use activitypub_federation::config::{Data, FederationConfig};
|
||||
use anyhow::anyhow;
|
||||
use lemmy_db_schema::{
|
||||
source::secret::Secret,
|
||||
utils::{ActualDbPool, DbPool},
|
||||
utils::{build_db_pool_for_tests, ActualDbPool, DbPool},
|
||||
};
|
||||
use lemmy_utils::{
|
||||
rate_limit::RateLimitCell,
|
||||
settings::{structs::Settings, SETTINGS},
|
||||
};
|
||||
use reqwest_middleware::ClientWithMiddleware;
|
||||
use reqwest::{Request, Response};
|
||||
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, Middleware, Next};
|
||||
use std::sync::Arc;
|
||||
use task_local_extensions::Extensions;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LemmyContext {
|
||||
|
@ -49,4 +54,62 @@ impl LemmyContext {
|
|||
pub fn rate_limit_cell(&self) -> &RateLimitCell {
|
||||
&self.rate_limit_cell
|
||||
}
|
||||
|
||||
/// Initialize a context for use in tests, optionally blocks network requests.
|
||||
///
|
||||
/// Do not use this in production code.
|
||||
pub async fn init_test_context() -> Data<LemmyContext> {
|
||||
Self::build_test_context(true).await
|
||||
}
|
||||
|
||||
/// Initialize a context for use in tests, with network requests allowed.
|
||||
/// TODO: get rid of this if possible.
|
||||
///
|
||||
/// Do not use this in production code.
|
||||
pub async fn init_test_context_with_networking() -> Data<LemmyContext> {
|
||||
Self::build_test_context(false).await
|
||||
}
|
||||
|
||||
async fn build_test_context(block_networking: bool) -> Data<LemmyContext> {
|
||||
// call this to run migrations
|
||||
let pool = build_db_pool_for_tests().await;
|
||||
|
||||
let client = client_builder(&SETTINGS).build().expect("build client");
|
||||
|
||||
let mut client = ClientBuilder::new(client);
|
||||
if block_networking {
|
||||
client = client.with(BlockedMiddleware);
|
||||
}
|
||||
let client = client.build();
|
||||
let secret = Secret {
|
||||
id: 0,
|
||||
jwt_secret: String::new(),
|
||||
};
|
||||
|
||||
let rate_limit_cell = RateLimitCell::with_test_config();
|
||||
|
||||
let context = LemmyContext::create(pool, client, secret, rate_limit_cell.clone());
|
||||
let config = FederationConfig::builder()
|
||||
.domain(context.settings().hostname.clone())
|
||||
.app_data(context)
|
||||
.build()
|
||||
.await
|
||||
.expect("build federation config");
|
||||
config.to_request_data()
|
||||
}
|
||||
}
|
||||
|
||||
struct BlockedMiddleware;
|
||||
|
||||
/// A reqwest middleware which blocks all requests
|
||||
#[async_trait::async_trait]
|
||||
impl Middleware for BlockedMiddleware {
|
||||
async fn handle(
|
||||
&self,
|
||||
_req: Request,
|
||||
_extensions: &mut Extensions,
|
||||
_next: Next<'_>,
|
||||
) -> reqwest_middleware::Result<Response> {
|
||||
Err(anyhow!("Network requests not allowed").into())
|
||||
}
|
||||
}
|
||||
|
|
|
@ -238,15 +238,28 @@ pub struct GetSiteMetadata {
|
|||
#[cfg_attr(feature = "full", ts(export))]
|
||||
/// The site metadata response.
|
||||
pub struct GetSiteMetadataResponse {
|
||||
pub metadata: SiteMetadata,
|
||||
pub metadata: LinkMetadata,
|
||||
}
|
||||
|
||||
#[skip_serializing_none]
|
||||
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)]
|
||||
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone, Default)]
|
||||
#[cfg_attr(feature = "full", derive(TS))]
|
||||
#[cfg_attr(feature = "full", ts(export))]
|
||||
/// Site metadata, from its opengraph tags.
|
||||
pub struct SiteMetadata {
|
||||
pub struct LinkMetadata {
|
||||
#[serde(flatten)]
|
||||
pub opengraph_data: OpenGraphData,
|
||||
pub content_type: Option<String>,
|
||||
#[serde(skip)]
|
||||
pub thumbnail: Option<DbUrl>,
|
||||
}
|
||||
|
||||
#[skip_serializing_none]
|
||||
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone, Default)]
|
||||
#[cfg_attr(feature = "full", derive(TS))]
|
||||
#[cfg_attr(feature = "full", ts(export))]
|
||||
/// Site metadata, from its opengraph tags.
|
||||
pub struct OpenGraphData {
|
||||
pub title: Option<String>,
|
||||
pub description: Option<String>,
|
||||
pub(crate) image: Option<DbUrl>,
|
||||
|
|
|
@ -1,39 +1,91 @@
|
|||
use crate::{context::LemmyContext, post::SiteMetadata};
|
||||
use crate::{
|
||||
context::LemmyContext,
|
||||
post::{LinkMetadata, OpenGraphData},
|
||||
utils::proxy_image_link,
|
||||
};
|
||||
use encoding::{all::encodings, DecoderTrap};
|
||||
use lemmy_db_schema::newtypes::DbUrl;
|
||||
use lemmy_utils::{
|
||||
error::{LemmyError, LemmyErrorType},
|
||||
settings::structs::Settings,
|
||||
settings::structs::{PictrsImageMode, Settings},
|
||||
version::VERSION,
|
||||
REQWEST_TIMEOUT,
|
||||
};
|
||||
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
|
||||
use reqwest::{Client, ClientBuilder};
|
||||
use mime::Mime;
|
||||
use reqwest::{header::CONTENT_TYPE, Client, ClientBuilder};
|
||||
use reqwest_middleware::ClientWithMiddleware;
|
||||
use serde::Deserialize;
|
||||
use tracing::info;
|
||||
use url::Url;
|
||||
use urlencoding::encode;
|
||||
use webpage::HTML;
|
||||
|
||||
/// Fetches the post link html tags (like title, description, image, etc)
|
||||
pub fn client_builder(settings: &Settings) -> ClientBuilder {
|
||||
let user_agent = format!(
|
||||
"Lemmy/{}; +{}",
|
||||
VERSION,
|
||||
settings.get_protocol_and_hostname()
|
||||
);
|
||||
|
||||
Client::builder()
|
||||
.user_agent(user_agent.clone())
|
||||
.timeout(REQWEST_TIMEOUT)
|
||||
.connect_timeout(REQWEST_TIMEOUT)
|
||||
}
|
||||
|
||||
/// Fetches metadata for the given link and optionally generates thumbnail.
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn fetch_site_metadata(
|
||||
client: &ClientWithMiddleware,
|
||||
pub async fn fetch_link_metadata(
|
||||
url: &Url,
|
||||
) -> Result<SiteMetadata, LemmyError> {
|
||||
generate_thumbnail: bool,
|
||||
context: &LemmyContext,
|
||||
) -> Result<LinkMetadata, LemmyError> {
|
||||
info!("Fetching site metadata for url: {}", url);
|
||||
let response = client.get(url.as_str()).send().await?;
|
||||
let response = context.client().get(url.as_str()).send().await?;
|
||||
|
||||
let content_type: Option<Mime> = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|h| h.to_str().ok())
|
||||
.and_then(|h| h.parse().ok());
|
||||
|
||||
// Can't use .text() here, because it only checks the content header, not the actual bytes
|
||||
// https://github.com/LemmyNet/lemmy/issues/1964
|
||||
let html_bytes = response.bytes().await.map_err(LemmyError::from)?.to_vec();
|
||||
|
||||
let tags = html_to_site_metadata(&html_bytes, url)?;
|
||||
let opengraph_data = extract_opengraph_data(&html_bytes, url).unwrap_or_default();
|
||||
let thumbnail = extract_thumbnail_from_opengraph_data(
|
||||
url,
|
||||
&opengraph_data,
|
||||
&content_type,
|
||||
generate_thumbnail,
|
||||
context,
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(tags)
|
||||
Ok(LinkMetadata {
|
||||
opengraph_data,
|
||||
content_type: content_type.map(|c| c.to_string()),
|
||||
thumbnail,
|
||||
})
|
||||
}
|
||||
|
||||
fn html_to_site_metadata(html_bytes: &[u8], url: &Url) -> Result<SiteMetadata, LemmyError> {
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn fetch_link_metadata_opt(
|
||||
url: Option<&Url>,
|
||||
generate_thumbnail: bool,
|
||||
context: &LemmyContext,
|
||||
) -> LinkMetadata {
|
||||
match &url {
|
||||
Some(url) => fetch_link_metadata(url, generate_thumbnail, context)
|
||||
.await
|
||||
.unwrap_or_default(),
|
||||
_ => Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract site metadata from HTML Opengraph attributes.
|
||||
fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> Result<OpenGraphData, LemmyError> {
|
||||
let html = String::from_utf8_lossy(html_bytes);
|
||||
|
||||
// Make sure the first line is doctype html
|
||||
|
@ -89,7 +141,7 @@ fn html_to_site_metadata(html_bytes: &[u8], url: &Url) -> Result<SiteMetadata, L
|
|||
// join also works if the target URL is absolute
|
||||
.and_then(|v| url.join(&v.url).ok());
|
||||
|
||||
Ok(SiteMetadata {
|
||||
Ok(OpenGraphData {
|
||||
title: og_title.or(page_title),
|
||||
description: og_description.or(page_description),
|
||||
image: og_image.map(Into::into),
|
||||
|
@ -97,59 +149,48 @@ fn html_to_site_metadata(html_bytes: &[u8], url: &Url) -> Result<SiteMetadata, L
|
|||
})
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub(crate) struct PictrsResponse {
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn extract_thumbnail_from_opengraph_data(
|
||||
url: &Url,
|
||||
opengraph_data: &OpenGraphData,
|
||||
content_type: &Option<Mime>,
|
||||
generate_thumbnail: bool,
|
||||
context: &LemmyContext,
|
||||
) -> Option<DbUrl> {
|
||||
let is_image = content_type.as_ref().unwrap_or(&mime::TEXT_PLAIN).type_() == mime::IMAGE;
|
||||
if generate_thumbnail && is_image {
|
||||
let image_url = opengraph_data
|
||||
.image
|
||||
.as_ref()
|
||||
.map(lemmy_db_schema::newtypes::DbUrl::inner)
|
||||
.unwrap_or(url);
|
||||
generate_pictrs_thumbnail(image_url, context)
|
||||
.await
|
||||
.ok()
|
||||
.map(Into::into)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct PictrsResponse {
|
||||
files: Vec<PictrsFile>,
|
||||
msg: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub(crate) struct PictrsFile {
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct PictrsFile {
|
||||
file: String,
|
||||
#[allow(dead_code)]
|
||||
delete_token: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub(crate) struct PictrsPurgeResponse {
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct PictrsPurgeResponse {
|
||||
msg: String,
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub(crate) async fn fetch_pictrs(
|
||||
client: &ClientWithMiddleware,
|
||||
settings: &Settings,
|
||||
image_url: &Url,
|
||||
) -> Result<PictrsResponse, LemmyError> {
|
||||
let pictrs_config = settings.pictrs_config()?;
|
||||
is_image_content_type(client, image_url).await?;
|
||||
|
||||
if pictrs_config.cache_external_link_previews {
|
||||
// fetch remote non-pictrs images for persistent thumbnail link
|
||||
let fetch_url = format!(
|
||||
"{}image/download?url={}",
|
||||
pictrs_config.url,
|
||||
utf8_percent_encode(image_url.as_str(), NON_ALPHANUMERIC) // TODO this might not be needed
|
||||
);
|
||||
|
||||
let response = client
|
||||
.get(&fetch_url)
|
||||
.timeout(REQWEST_TIMEOUT)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let response: PictrsResponse = response.json().await.map_err(LemmyError::from)?;
|
||||
|
||||
if response.msg == "ok" {
|
||||
Ok(response)
|
||||
} else {
|
||||
Err(LemmyErrorType::PictrsResponseError(response.msg))?
|
||||
}
|
||||
} else {
|
||||
Err(LemmyErrorType::PictrsCachingDisabled)?
|
||||
}
|
||||
}
|
||||
|
||||
/// Purges an image from pictrs
|
||||
/// Note: This should often be coerced from a Result to .ok() in order to fail softly, because:
|
||||
/// - It might fail due to image being not local
|
||||
|
@ -167,13 +208,6 @@ pub async fn purge_image_from_pictrs(
|
|||
.next_back()
|
||||
.ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?;
|
||||
|
||||
purge_image_from_pictrs_by_alias(alias, context).await
|
||||
}
|
||||
|
||||
pub async fn purge_image_from_pictrs_by_alias(
|
||||
alias: &str,
|
||||
context: &LemmyContext,
|
||||
) -> Result<(), LemmyError> {
|
||||
let pictrs_config = context.settings().pictrs_config()?;
|
||||
let purge_url = format!("{}internal/purge?alias={}", pictrs_config.url, alias);
|
||||
|
||||
|
@ -190,10 +224,9 @@ pub async fn purge_image_from_pictrs_by_alias(
|
|||
|
||||
let response: PictrsPurgeResponse = response.json().await.map_err(LemmyError::from)?;
|
||||
|
||||
if response.msg == "ok" {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(LemmyErrorType::PictrsPurgeResponseError(response.msg))?
|
||||
match response.msg.as_str() {
|
||||
"ok" => Ok(()),
|
||||
_ => Err(LemmyErrorType::PictrsPurgeResponseError(response.msg))?,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -217,62 +250,48 @@ pub async fn delete_image_from_pictrs(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Both are options, since the URL might be either an html page, or an image
|
||||
/// Returns the SiteMetadata, and an image URL, if there is a picture associated
|
||||
/// Retrieves the image with local pict-rs and generates a thumbnail. Returns the thumbnail url.
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn fetch_site_data(
|
||||
client: &ClientWithMiddleware,
|
||||
settings: &Settings,
|
||||
url: Option<&Url>,
|
||||
include_image: bool,
|
||||
) -> (Option<SiteMetadata>, Option<DbUrl>) {
|
||||
match &url {
|
||||
Some(url) => {
|
||||
// Fetch metadata
|
||||
// Ignore errors, since it may be an image, or not have the data.
|
||||
// Warning, this may ignore SSL errors
|
||||
let metadata_option = fetch_site_metadata(client, url).await.ok();
|
||||
if !include_image {
|
||||
(metadata_option, None)
|
||||
} else {
|
||||
let thumbnail_url =
|
||||
fetch_pictrs_url_from_site_metadata(client, &metadata_option, settings, url)
|
||||
.await
|
||||
.ok();
|
||||
(metadata_option, thumbnail_url)
|
||||
}
|
||||
}
|
||||
None => (None, None),
|
||||
async fn generate_pictrs_thumbnail(
|
||||
image_url: &Url,
|
||||
context: &LemmyContext,
|
||||
) -> Result<Url, LemmyError> {
|
||||
let pictrs_config = context.settings().pictrs_config()?;
|
||||
|
||||
if pictrs_config.image_mode() == PictrsImageMode::ProxyAllImages {
|
||||
return Ok(proxy_image_link(image_url.clone(), context).await?.into());
|
||||
}
|
||||
|
||||
// fetch remote non-pictrs images for persistent thumbnail link
|
||||
// TODO: should limit size once supported by pictrs
|
||||
let fetch_url = format!(
|
||||
"{}image/download?url={}",
|
||||
pictrs_config.url,
|
||||
encode(image_url.as_str())
|
||||
);
|
||||
|
||||
let response = context
|
||||
.client()
|
||||
.get(&fetch_url)
|
||||
.timeout(REQWEST_TIMEOUT)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let response: PictrsResponse = response.json().await?;
|
||||
|
||||
if response.msg == "ok" {
|
||||
let thumbnail_url = Url::parse(&format!(
|
||||
"{}/pictrs/image/{}",
|
||||
context.settings().get_protocol_and_hostname(),
|
||||
response.files.first().expect("missing pictrs file").file
|
||||
))?;
|
||||
Ok(thumbnail_url)
|
||||
} else {
|
||||
Err(LemmyErrorType::PictrsResponseError(response.msg))?
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_pictrs_url_from_site_metadata(
|
||||
client: &ClientWithMiddleware,
|
||||
metadata_option: &Option<SiteMetadata>,
|
||||
settings: &Settings,
|
||||
url: &Url,
|
||||
) -> Result<DbUrl, LemmyError> {
|
||||
let pictrs_res = match metadata_option {
|
||||
Some(metadata_res) => match &metadata_res.image {
|
||||
// Metadata, with image
|
||||
// Try to generate a small thumbnail if there's a full sized one from post-links
|
||||
Some(metadata_image) => fetch_pictrs(client, settings, metadata_image).await,
|
||||
// Metadata, but no image
|
||||
None => fetch_pictrs(client, settings, url).await,
|
||||
},
|
||||
// No metadata, try to fetch the URL as an image
|
||||
None => fetch_pictrs(client, settings, url).await,
|
||||
}?;
|
||||
|
||||
Url::parse(&format!(
|
||||
"{}/pictrs/image/{}",
|
||||
settings.get_protocol_and_hostname(),
|
||||
pictrs_res.files.first().expect("missing pictrs file").file
|
||||
))
|
||||
.map(Into::into)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
// TODO: get rid of this by reading content type from db
|
||||
#[tracing::instrument(skip_all)]
|
||||
async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Result<(), LemmyError> {
|
||||
let response = client.get(url.as_str()).send().await?;
|
||||
|
@ -289,51 +308,50 @@ async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Resu
|
|||
}
|
||||
}
|
||||
|
||||
pub fn client_builder(settings: &Settings) -> ClientBuilder {
|
||||
let user_agent = format!(
|
||||
"Lemmy/{}; +{}",
|
||||
VERSION,
|
||||
settings.get_protocol_and_hostname()
|
||||
);
|
||||
|
||||
Client::builder()
|
||||
.user_agent(user_agent)
|
||||
.timeout(REQWEST_TIMEOUT)
|
||||
.connect_timeout(REQWEST_TIMEOUT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used)]
|
||||
#![allow(clippy::indexing_slicing)]
|
||||
|
||||
use crate::request::{client_builder, fetch_site_metadata, html_to_site_metadata, SiteMetadata};
|
||||
use lemmy_utils::settings::SETTINGS;
|
||||
use crate::{
|
||||
context::LemmyContext,
|
||||
request::{extract_opengraph_data, fetch_link_metadata},
|
||||
};
|
||||
use pretty_assertions::assert_eq;
|
||||
use serial_test::serial;
|
||||
use url::Url;
|
||||
|
||||
// These helped with testing
|
||||
#[tokio::test]
|
||||
async fn test_site_metadata() {
|
||||
let settings = &SETTINGS.clone();
|
||||
let client = client_builder(settings).build().unwrap().into();
|
||||
#[serial]
|
||||
async fn test_link_metadata() {
|
||||
let context = LemmyContext::init_test_context_with_networking().await;
|
||||
let sample_url = Url::parse("https://gitlab.com/IzzyOnDroid/repo/-/wikis/FAQ").unwrap();
|
||||
let sample_res = fetch_site_metadata(&client, &sample_url).await.unwrap();
|
||||
let sample_res = fetch_link_metadata(&sample_url, false, &context)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
SiteMetadata {
|
||||
title: Some("FAQ · Wiki · IzzyOnDroid / repo · GitLab".to_string()),
|
||||
description: Some(
|
||||
"The F-Droid compatible repo at https://apt.izzysoft.de/fdroid/".to_string()
|
||||
),
|
||||
image: Some(
|
||||
Url::parse("https://gitlab.com/uploads/-/system/project/avatar/4877469/iod_logo.png")
|
||||
.unwrap()
|
||||
.into()
|
||||
),
|
||||
embed_video_url: None,
|
||||
},
|
||||
sample_res
|
||||
Some("FAQ · Wiki · IzzyOnDroid / repo · GitLab".to_string()),
|
||||
sample_res.opengraph_data.title
|
||||
);
|
||||
assert_eq!(
|
||||
Some("The F-Droid compatible repo at https://apt.izzysoft.de/fdroid/".to_string()),
|
||||
sample_res.opengraph_data.description
|
||||
);
|
||||
assert_eq!(
|
||||
Some(
|
||||
Url::parse("https://gitlab.com/uploads/-/system/project/avatar/4877469/iod_logo.png")
|
||||
.unwrap()
|
||||
.into()
|
||||
),
|
||||
sample_res.opengraph_data.image
|
||||
);
|
||||
assert_eq!(None, sample_res.opengraph_data.embed_video_url);
|
||||
assert_eq!(
|
||||
Some(mime::TEXT_HTML_UTF_8.to_string()),
|
||||
sample_res.content_type
|
||||
);
|
||||
assert_eq!(None, sample_res.thumbnail);
|
||||
}
|
||||
|
||||
// #[test]
|
||||
|
@ -351,7 +369,7 @@ mod tests {
|
|||
|
||||
// root relative url
|
||||
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='/image.jpg'></head><body></body></html>";
|
||||
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
|
||||
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
|
||||
assert_eq!(
|
||||
metadata.image,
|
||||
Some(Url::parse("https://example.com/image.jpg").unwrap().into())
|
||||
|
@ -359,7 +377,7 @@ mod tests {
|
|||
|
||||
// base relative url
|
||||
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='image.jpg'></head><body></body></html>";
|
||||
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
|
||||
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
|
||||
assert_eq!(
|
||||
metadata.image,
|
||||
Some(
|
||||
|
@ -371,7 +389,7 @@ mod tests {
|
|||
|
||||
// absolute url
|
||||
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='https://cdn.host.com/image.jpg'></head><body></body></html>";
|
||||
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
|
||||
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
|
||||
assert_eq!(
|
||||
metadata.image,
|
||||
Some(Url::parse("https://cdn.host.com/image.jpg").unwrap().into())
|
||||
|
@ -379,7 +397,7 @@ mod tests {
|
|||
|
||||
// protocol relative url
|
||||
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='//example.com/image.jpg'></head><body></body></html>";
|
||||
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
|
||||
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
|
||||
assert_eq!(
|
||||
metadata.image,
|
||||
Some(Url::parse("https://example.com/image.jpg").unwrap().into())
|
||||
|
|
|
@ -12,6 +12,7 @@ use lemmy_db_schema::{
|
|||
community::{Community, CommunityModerator, CommunityUpdateForm},
|
||||
community_block::CommunityBlock,
|
||||
email_verification::{EmailVerification, EmailVerificationForm},
|
||||
images::RemoteImage,
|
||||
instance::Instance,
|
||||
instance_block::InstanceBlock,
|
||||
local_site::LocalSite,
|
||||
|
@ -35,14 +36,18 @@ use lemmy_utils::{
|
|||
email::{send_email, translations::Lang},
|
||||
error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult},
|
||||
rate_limit::{ActionType, BucketConfig},
|
||||
settings::structs::Settings,
|
||||
utils::slurs::build_slur_regex,
|
||||
settings::structs::{PictrsImageMode, Settings},
|
||||
utils::{
|
||||
markdown::markdown_rewrite_image_links,
|
||||
slurs::{build_slur_regex, remove_slurs},
|
||||
},
|
||||
};
|
||||
use regex::Regex;
|
||||
use rosetta_i18n::{Language, LanguageId};
|
||||
use std::collections::HashSet;
|
||||
use tracing::warn;
|
||||
use url::{ParseError, Url};
|
||||
use urlencoding::encode;
|
||||
|
||||
pub static AUTH_COOKIE_NAME: &str = "jwt";
|
||||
|
||||
|
@ -848,14 +853,115 @@ fn limit_expire_time(expires: DateTime<Utc>) -> LemmyResult<Option<DateTime<Utc>
|
|||
}
|
||||
}
|
||||
|
||||
pub async fn process_markdown(
|
||||
text: &str,
|
||||
slur_regex: &Option<Regex>,
|
||||
context: &LemmyContext,
|
||||
) -> LemmyResult<String> {
|
||||
let text = remove_slurs(text, slur_regex);
|
||||
if context.settings().pictrs_config()?.image_mode() == PictrsImageMode::ProxyAllImages {
|
||||
let (text, links) = markdown_rewrite_image_links(text);
|
||||
RemoteImage::create(&mut context.pool(), links).await?;
|
||||
Ok(text)
|
||||
} else {
|
||||
Ok(text)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn process_markdown_opt(
|
||||
text: &Option<String>,
|
||||
slur_regex: &Option<Regex>,
|
||||
context: &LemmyContext,
|
||||
) -> LemmyResult<Option<String>> {
|
||||
match text {
|
||||
Some(t) => process_markdown(t, slur_regex, context).await.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// A wrapper for `proxy_image_link` for use in tests.
|
||||
///
|
||||
/// The parameter `force_image_proxy` is the config value of `pictrs.image_proxy`. Its necessary to pass
|
||||
/// as separate parameter so it can be changed in tests.
|
||||
async fn proxy_image_link_internal(
|
||||
link: Url,
|
||||
image_mode: PictrsImageMode,
|
||||
context: &LemmyContext,
|
||||
) -> LemmyResult<DbUrl> {
|
||||
// Dont rewrite links pointing to local domain.
|
||||
if link.domain() == Some(&context.settings().hostname) {
|
||||
Ok(link.into())
|
||||
} else if image_mode == PictrsImageMode::ProxyAllImages {
|
||||
let proxied = format!(
|
||||
"{}/api/v3/image_proxy?url={}",
|
||||
context.settings().get_protocol_and_hostname(),
|
||||
encode(link.as_str())
|
||||
);
|
||||
RemoteImage::create(&mut context.pool(), vec![link]).await?;
|
||||
Ok(Url::parse(&proxied)?.into())
|
||||
} else {
|
||||
Ok(link.into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Rewrite a link to go through `/api/v3/image_proxy` endpoint. This is only for remote urls and
|
||||
/// if image_proxy setting is enabled.
|
||||
pub(crate) async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult<DbUrl> {
|
||||
proxy_image_link_internal(
|
||||
link,
|
||||
context.settings().pictrs_config()?.image_mode(),
|
||||
context,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn proxy_image_link_opt_api(
|
||||
link: &Option<String>,
|
||||
context: &LemmyContext,
|
||||
) -> LemmyResult<Option<Option<DbUrl>>> {
|
||||
proxy_image_link_api(link, context).await.map(Some)
|
||||
}
|
||||
|
||||
pub async fn proxy_image_link_api(
|
||||
link: &Option<String>,
|
||||
context: &LemmyContext,
|
||||
) -> LemmyResult<Option<DbUrl>> {
|
||||
let link: Option<DbUrl> = match link.as_ref().map(String::as_str) {
|
||||
// An empty string is an erase
|
||||
Some("") => None,
|
||||
Some(str_url) => Url::parse(str_url)
|
||||
.map(|u| Some(u.into()))
|
||||
.with_lemmy_type(LemmyErrorType::InvalidUrl)?,
|
||||
None => None,
|
||||
};
|
||||
if let Some(l) = link {
|
||||
proxy_image_link(l.into(), context).await.map(Some)
|
||||
} else {
|
||||
Ok(link)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn proxy_image_link_opt_apub(
|
||||
link: Option<Url>,
|
||||
context: &LemmyContext,
|
||||
) -> LemmyResult<Option<DbUrl>> {
|
||||
if let Some(l) = link {
|
||||
proxy_image_link(l, context).await.map(Some)
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used)]
|
||||
#![allow(clippy::indexing_slicing)]
|
||||
|
||||
use super::*;
|
||||
use crate::utils::{honeypot_check, limit_expire_time, password_length_check};
|
||||
use chrono::{Days, Utc};
|
||||
use pretty_assertions::assert_eq;
|
||||
use serial_test::serial;
|
||||
|
||||
#[test]
|
||||
#[rustfmt::skip]
|
||||
|
@ -894,4 +1000,62 @@ mod tests {
|
|||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_proxy_image_link() {
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
|
||||
// image from local domain is unchanged
|
||||
let local_url = Url::parse("http://lemmy-alpha/image.png").unwrap();
|
||||
let proxied =
|
||||
proxy_image_link_internal(local_url.clone(), PictrsImageMode::ProxyAllImages, &context)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(&local_url, proxied.inner());
|
||||
|
||||
// image from remote domain is proxied
|
||||
let remote_image = Url::parse("http://lemmy-beta/image.png").unwrap();
|
||||
let proxied = proxy_image_link_internal(
|
||||
remote_image.clone(),
|
||||
PictrsImageMode::ProxyAllImages,
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
"https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Flemmy-beta%2Fimage.png",
|
||||
proxied.as_str()
|
||||
);
|
||||
assert!(
|
||||
RemoteImage::validate(&mut context.pool(), remote_image.into())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_diesel_option_overwrite_to_url() {
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
|
||||
assert!(matches!(
|
||||
proxy_image_link_api(&None, &context).await,
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
proxy_image_link_opt_api(&Some(String::new()), &context).await,
|
||||
Ok(Some(None))
|
||||
));
|
||||
assert!(
|
||||
proxy_image_link_opt_api(&Some("invalid_url".to_string()), &context)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let example_url = "https://lemmy-alpha/image.png";
|
||||
assert!(matches!(
|
||||
proxy_image_link_opt_api(&Some(example_url.to_string()), &context).await,
|
||||
Ok(Some(Some(url))) if url == Url::parse(example_url).unwrap().into()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,6 +11,7 @@ use lemmy_api_common::{
|
|||
generate_local_apub_endpoint,
|
||||
get_post,
|
||||
local_site_to_slur_regex,
|
||||
process_markdown,
|
||||
EndpointType,
|
||||
},
|
||||
};
|
||||
|
@ -28,11 +29,7 @@ use lemmy_db_schema::{
|
|||
use lemmy_db_views::structs::LocalUserView;
|
||||
use lemmy_utils::{
|
||||
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
|
||||
utils::{
|
||||
mention::scrape_text_for_mentions,
|
||||
slurs::remove_slurs,
|
||||
validation::is_valid_body_field,
|
||||
},
|
||||
utils::{mention::scrape_text_for_mentions, validation::is_valid_body_field},
|
||||
};
|
||||
|
||||
const MAX_COMMENT_DEPTH_LIMIT: usize = 100;
|
||||
|
@ -45,10 +42,8 @@ pub async fn create_comment(
|
|||
) -> Result<Json<CommentResponse>, LemmyError> {
|
||||
let local_site = LocalSite::read(&mut context.pool()).await?;
|
||||
|
||||
let content = remove_slurs(
|
||||
&data.content.clone(),
|
||||
&local_site_to_slur_regex(&local_site),
|
||||
);
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
let content = process_markdown(&data.content, &slur_regex, &context).await?;
|
||||
is_valid_body_field(&Some(content.clone()), false)?;
|
||||
|
||||
// Check for a community ban
|
||||
|
|
|
@ -5,7 +5,7 @@ use lemmy_api_common::{
|
|||
comment::{CommentResponse, EditComment},
|
||||
context::LemmyContext,
|
||||
send_activity::{ActivityChannel, SendActivityData},
|
||||
utils::{check_community_user_action, local_site_to_slur_regex},
|
||||
utils::{check_community_user_action, local_site_to_slur_regex, process_markdown_opt},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
|
@ -19,11 +19,7 @@ use lemmy_db_schema::{
|
|||
use lemmy_db_views::structs::{CommentView, LocalUserView};
|
||||
use lemmy_utils::{
|
||||
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
|
||||
utils::{
|
||||
mention::scrape_text_for_mentions,
|
||||
slurs::remove_slurs,
|
||||
validation::is_valid_body_field,
|
||||
},
|
||||
utils::{mention::scrape_text_for_mentions, validation::is_valid_body_field},
|
||||
};
|
||||
|
||||
#[tracing::instrument(skip(context))]
|
||||
|
@ -57,11 +53,8 @@ pub async fn update_comment(
|
|||
)
|
||||
.await?;
|
||||
|
||||
// Update the Content
|
||||
let content = data
|
||||
.content
|
||||
.as_ref()
|
||||
.map(|c| remove_slurs(c, &local_site_to_slur_regex(&local_site)));
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
let content = process_markdown_opt(&data.content, &slur_regex, &context).await?;
|
||||
is_valid_body_field(&content, false)?;
|
||||
|
||||
let comment_id = data.comment_id;
|
||||
|
|
|
@ -11,6 +11,8 @@ use lemmy_api_common::{
|
|||
generate_shared_inbox_url,
|
||||
is_admin,
|
||||
local_site_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_api,
|
||||
EndpointType,
|
||||
},
|
||||
};
|
||||
|
@ -27,13 +29,12 @@ use lemmy_db_schema::{
|
|||
},
|
||||
},
|
||||
traits::{ApubActor, Crud, Followable, Joinable},
|
||||
utils::diesel_option_overwrite_to_url_create,
|
||||
};
|
||||
use lemmy_db_views::structs::{LocalUserView, SiteView};
|
||||
use lemmy_utils::{
|
||||
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
|
||||
utils::{
|
||||
slurs::{check_slurs, check_slurs_opt},
|
||||
slurs::check_slurs,
|
||||
validation::{is_valid_actor_name, is_valid_body_field},
|
||||
},
|
||||
};
|
||||
|
@ -51,14 +52,12 @@ pub async fn create_community(
|
|||
Err(LemmyErrorType::OnlyAdminsCanCreateCommunities)?
|
||||
}
|
||||
|
||||
// Check to make sure the icon and banners are urls
|
||||
let icon = diesel_option_overwrite_to_url_create(&data.icon)?;
|
||||
let banner = diesel_option_overwrite_to_url_create(&data.banner)?;
|
||||
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
check_slurs(&data.name, &slur_regex)?;
|
||||
check_slurs(&data.title, &slur_regex)?;
|
||||
check_slurs_opt(&data.description, &slur_regex)?;
|
||||
let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
|
||||
let icon = proxy_image_link_api(&data.icon, &context).await?;
|
||||
let banner = proxy_image_link_api(&data.banner, &context).await?;
|
||||
|
||||
is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?;
|
||||
is_valid_body_field(&data.description, false)?;
|
||||
|
@ -81,7 +80,7 @@ pub async fn create_community(
|
|||
let community_form = CommunityInsertForm::builder()
|
||||
.name(data.name.clone())
|
||||
.title(data.title.clone())
|
||||
.description(data.description.clone())
|
||||
.description(description)
|
||||
.icon(icon)
|
||||
.banner(banner)
|
||||
.nsfw(data.nsfw)
|
||||
|
|
|
@ -5,7 +5,12 @@ use lemmy_api_common::{
|
|||
community::{CommunityResponse, EditCommunity},
|
||||
context::LemmyContext,
|
||||
send_activity::{ActivityChannel, SendActivityData},
|
||||
utils::{check_community_mod_action, local_site_to_slur_regex},
|
||||
utils::{
|
||||
check_community_mod_action,
|
||||
local_site_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_api,
|
||||
},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
|
@ -14,7 +19,7 @@ use lemmy_db_schema::{
|
|||
local_site::LocalSite,
|
||||
},
|
||||
traits::Crud,
|
||||
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
|
||||
utils::{diesel_option_overwrite, naive_now},
|
||||
};
|
||||
use lemmy_db_views::structs::LocalUserView;
|
||||
use lemmy_utils::{
|
||||
|
@ -32,12 +37,12 @@ pub async fn update_community(
|
|||
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
check_slurs_opt(&data.title, &slur_regex)?;
|
||||
check_slurs_opt(&data.description, &slur_regex)?;
|
||||
let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
|
||||
is_valid_body_field(&data.description, false)?;
|
||||
|
||||
let icon = diesel_option_overwrite_to_url(&data.icon)?;
|
||||
let banner = diesel_option_overwrite_to_url(&data.banner)?;
|
||||
let description = diesel_option_overwrite(data.description.clone());
|
||||
let description = diesel_option_overwrite(description);
|
||||
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
|
||||
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
|
||||
|
||||
// Verify its a mod (only mods can edit it)
|
||||
check_community_mod_action(
|
||||
|
|
|
@ -4,7 +4,7 @@ use lemmy_api_common::{
|
|||
build_response::build_post_response,
|
||||
context::LemmyContext,
|
||||
post::{CreatePost, PostResponse},
|
||||
request::fetch_site_data,
|
||||
request::fetch_link_metadata_opt,
|
||||
send_activity::{ActivityChannel, SendActivityData},
|
||||
utils::{
|
||||
check_community_user_action,
|
||||
|
@ -12,6 +12,8 @@ use lemmy_api_common::{
|
|||
honeypot_check,
|
||||
local_site_to_slur_regex,
|
||||
mark_post_as_read,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_apub,
|
||||
EndpointType,
|
||||
},
|
||||
};
|
||||
|
@ -31,7 +33,7 @@ use lemmy_utils::{
|
|||
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
|
||||
spawn_try_task,
|
||||
utils::{
|
||||
slurs::{check_slurs, check_slurs_opt},
|
||||
slurs::check_slurs,
|
||||
validation::{check_url_scheme, clean_url_params, is_valid_body_field, is_valid_post_title},
|
||||
},
|
||||
};
|
||||
|
@ -49,14 +51,14 @@ pub async fn create_post(
|
|||
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
check_slurs(&data.name, &slur_regex)?;
|
||||
check_slurs_opt(&data.body, &slur_regex)?;
|
||||
let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
|
||||
honeypot_check(&data.honeypot)?;
|
||||
|
||||
let data_url = data.url.as_ref();
|
||||
let url = data_url.map(clean_url_params).map(Into::into); // TODO no good way to handle a "clear"
|
||||
let url = data_url.map(clean_url_params); // TODO no good way to handle a "clear"
|
||||
|
||||
is_valid_post_title(&data.name)?;
|
||||
is_valid_body_field(&data.body, true)?;
|
||||
is_valid_body_field(&body, true)?;
|
||||
check_url_scheme(&data.url)?;
|
||||
|
||||
check_community_user_action(
|
||||
|
@ -82,11 +84,8 @@ pub async fn create_post(
|
|||
}
|
||||
|
||||
// Fetch post links and pictrs cached image
|
||||
let (metadata_res, thumbnail_url) =
|
||||
fetch_site_data(context.client(), context.settings(), data_url, true).await;
|
||||
let (embed_title, embed_description, embed_video_url) = metadata_res
|
||||
.map(|u| (u.title, u.description, u.embed_video_url))
|
||||
.unwrap_or_default();
|
||||
let metadata = fetch_link_metadata_opt(url.as_ref(), true, &context).await;
|
||||
let url = proxy_image_link_opt_apub(url, &context).await?;
|
||||
|
||||
// Only need to check if language is allowed in case user set it explicitly. When using default
|
||||
// language, it already only returns allowed languages.
|
||||
|
@ -113,15 +112,15 @@ pub async fn create_post(
|
|||
let post_form = PostInsertForm::builder()
|
||||
.name(data.name.trim().to_string())
|
||||
.url(url)
|
||||
.body(data.body.clone())
|
||||
.body(body)
|
||||
.community_id(data.community_id)
|
||||
.creator_id(local_user_view.person.id)
|
||||
.nsfw(data.nsfw)
|
||||
.embed_title(embed_title)
|
||||
.embed_description(embed_description)
|
||||
.embed_video_url(embed_video_url)
|
||||
.embed_title(metadata.opengraph_data.title)
|
||||
.embed_description(metadata.opengraph_data.description)
|
||||
.embed_video_url(metadata.opengraph_data.embed_video_url)
|
||||
.language_id(language_id)
|
||||
.thumbnail_url(thumbnail_url)
|
||||
.thumbnail_url(metadata.thumbnail)
|
||||
.build();
|
||||
|
||||
let inserted_post = Post::create(&mut context.pool(), &post_form)
|
||||
|
|
|
@ -4,9 +4,14 @@ use lemmy_api_common::{
|
|||
build_response::build_post_response,
|
||||
context::LemmyContext,
|
||||
post::{EditPost, PostResponse},
|
||||
request::fetch_site_data,
|
||||
request::fetch_link_metadata,
|
||||
send_activity::{ActivityChannel, SendActivityData},
|
||||
utils::{check_community_user_action, local_site_to_slur_regex},
|
||||
utils::{
|
||||
check_community_user_action,
|
||||
local_site_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_apub,
|
||||
},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
|
@ -35,21 +40,19 @@ pub async fn update_post(
|
|||
) -> Result<Json<PostResponse>, LemmyError> {
|
||||
let local_site = LocalSite::read(&mut context.pool()).await?;
|
||||
|
||||
let data_url = data.url.as_ref();
|
||||
|
||||
// TODO No good way to handle a clear.
|
||||
// Issue link: https://github.com/LemmyNet/lemmy/issues/2287
|
||||
let url = Some(data_url.map(clean_url_params).map(Into::into));
|
||||
let url = data.url.as_ref().map(clean_url_params);
|
||||
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
check_slurs_opt(&data.name, &slur_regex)?;
|
||||
check_slurs_opt(&data.body, &slur_regex)?;
|
||||
let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
|
||||
|
||||
if let Some(name) = &data.name {
|
||||
is_valid_post_title(name)?;
|
||||
}
|
||||
|
||||
is_valid_body_field(&data.body, true)?;
|
||||
is_valid_body_field(&body, true)?;
|
||||
check_url_scheme(&data.url)?;
|
||||
|
||||
let post_id = data.post_id;
|
||||
|
@ -67,13 +70,23 @@ pub async fn update_post(
|
|||
Err(LemmyErrorType::NoPostEditAllowed)?
|
||||
}
|
||||
|
||||
// Fetch post links and Pictrs cached image
|
||||
let data_url = data.url.as_ref();
|
||||
let (metadata_res, thumbnail_url) =
|
||||
fetch_site_data(context.client(), context.settings(), data_url, true).await;
|
||||
let (embed_title, embed_description, embed_video_url) = metadata_res
|
||||
.map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
|
||||
.unwrap_or_default();
|
||||
// Fetch post links and Pictrs cached image if url was updated
|
||||
let (embed_title, embed_description, embed_video_url, thumbnail_url) = match &url {
|
||||
Some(url) => {
|
||||
let metadata = fetch_link_metadata(url, true, &context).await?;
|
||||
(
|
||||
Some(metadata.opengraph_data.title),
|
||||
Some(metadata.opengraph_data.description),
|
||||
Some(metadata.opengraph_data.embed_video_url),
|
||||
Some(metadata.thumbnail),
|
||||
)
|
||||
}
|
||||
_ => Default::default(),
|
||||
};
|
||||
let url = match url {
|
||||
Some(url) => Some(proxy_image_link_opt_apub(Some(url), &context).await?),
|
||||
_ => Default::default(),
|
||||
};
|
||||
|
||||
let language_id = data.language_id;
|
||||
CommunityLanguage::is_allowed_community_language(
|
||||
|
@ -86,13 +99,13 @@ pub async fn update_post(
|
|||
let post_form = PostUpdateForm {
|
||||
name: data.name.clone(),
|
||||
url,
|
||||
body: diesel_option_overwrite(data.body.clone()),
|
||||
body: diesel_option_overwrite(body),
|
||||
nsfw: data.nsfw,
|
||||
embed_title,
|
||||
embed_description,
|
||||
embed_video_url,
|
||||
language_id: data.language_id,
|
||||
thumbnail_url: Some(thumbnail_url),
|
||||
thumbnail_url,
|
||||
updated: Some(Some(naive_now())),
|
||||
..Default::default()
|
||||
};
|
||||
|
|
|
@ -9,6 +9,7 @@ use lemmy_api_common::{
|
|||
generate_local_apub_endpoint,
|
||||
get_interface_language,
|
||||
local_site_to_slur_regex,
|
||||
process_markdown,
|
||||
send_email_to_user,
|
||||
EndpointType,
|
||||
},
|
||||
|
@ -23,7 +24,7 @@ use lemmy_db_schema::{
|
|||
use lemmy_db_views::structs::{LocalUserView, PrivateMessageView};
|
||||
use lemmy_utils::{
|
||||
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
|
||||
utils::{markdown::markdown_to_html, slurs::remove_slurs, validation::is_valid_body_field},
|
||||
utils::{markdown::markdown_to_html, validation::is_valid_body_field},
|
||||
};
|
||||
|
||||
#[tracing::instrument(skip(context))]
|
||||
|
@ -34,7 +35,8 @@ pub async fn create_private_message(
|
|||
) -> Result<Json<PrivateMessageResponse>, LemmyError> {
|
||||
let local_site = LocalSite::read(&mut context.pool()).await?;
|
||||
|
||||
let content = remove_slurs(&data.content, &local_site_to_slur_regex(&local_site));
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
let content = process_markdown(&data.content, &slur_regex, &context).await?;
|
||||
is_valid_body_field(&Some(content.clone()), false)?;
|
||||
|
||||
check_person_block(
|
||||
|
|
|
@ -4,7 +4,7 @@ use lemmy_api_common::{
|
|||
context::LemmyContext,
|
||||
private_message::{EditPrivateMessage, PrivateMessageResponse},
|
||||
send_activity::{ActivityChannel, SendActivityData},
|
||||
utils::local_site_to_slur_regex,
|
||||
utils::{local_site_to_slur_regex, process_markdown},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
|
@ -17,7 +17,7 @@ use lemmy_db_schema::{
|
|||
use lemmy_db_views::structs::{LocalUserView, PrivateMessageView};
|
||||
use lemmy_utils::{
|
||||
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
|
||||
utils::{slurs::remove_slurs, validation::is_valid_body_field},
|
||||
utils::validation::is_valid_body_field,
|
||||
};
|
||||
|
||||
#[tracing::instrument(skip(context))]
|
||||
|
@ -36,7 +36,8 @@ pub async fn update_private_message(
|
|||
}
|
||||
|
||||
// Doing the update
|
||||
let content = remove_slurs(&data.content, &local_site_to_slur_regex(&local_site));
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
let content = process_markdown(&data.content, &slur_regex, &context).await?;
|
||||
is_valid_body_field(&Some(content.clone()), false)?;
|
||||
|
||||
let private_message_id = data.private_message_id;
|
||||
|
|
|
@ -4,7 +4,14 @@ use actix_web::web::{Data, Json};
|
|||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
site::{CreateSite, SiteResponse},
|
||||
utils::{generate_shared_inbox_url, is_admin, local_site_rate_limit_to_rate_limit_config},
|
||||
utils::{
|
||||
generate_shared_inbox_url,
|
||||
is_admin,
|
||||
local_site_rate_limit_to_rate_limit_config,
|
||||
local_site_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_api,
|
||||
},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
newtypes::DbUrl,
|
||||
|
@ -15,7 +22,7 @@ use lemmy_db_schema::{
|
|||
tagline::Tagline,
|
||||
},
|
||||
traits::Crud,
|
||||
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
|
||||
utils::{diesel_option_overwrite, naive_now},
|
||||
};
|
||||
use lemmy_db_views::structs::{LocalUserView, SiteView};
|
||||
use lemmy_utils::{
|
||||
|
@ -50,12 +57,17 @@ pub async fn create_site(
|
|||
let inbox_url = Some(generate_shared_inbox_url(context.settings())?);
|
||||
let keypair = generate_actor_keypair()?;
|
||||
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
|
||||
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
|
||||
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
|
||||
|
||||
let site_form = SiteUpdateForm {
|
||||
name: Some(data.name.clone()),
|
||||
sidebar: diesel_option_overwrite(data.sidebar.clone()),
|
||||
sidebar: diesel_option_overwrite(sidebar),
|
||||
description: diesel_option_overwrite(data.description.clone()),
|
||||
icon: diesel_option_overwrite_to_url(&data.icon)?,
|
||||
banner: diesel_option_overwrite_to_url(&data.banner)?,
|
||||
icon,
|
||||
banner,
|
||||
actor_id: Some(actor_id),
|
||||
last_refreshed_at: Some(naive_now()),
|
||||
inbox_url,
|
||||
|
|
|
@ -3,7 +3,13 @@ use actix_web::web::{Data, Json};
|
|||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
site::{EditSite, SiteResponse},
|
||||
utils::{is_admin, local_site_rate_limit_to_rate_limit_config},
|
||||
utils::{
|
||||
is_admin,
|
||||
local_site_rate_limit_to_rate_limit_config,
|
||||
local_site_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_api,
|
||||
},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
|
@ -17,7 +23,7 @@ use lemmy_db_schema::{
|
|||
tagline::Tagline,
|
||||
},
|
||||
traits::Crud,
|
||||
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
|
||||
utils::{diesel_option_overwrite, naive_now},
|
||||
RegistrationMode,
|
||||
};
|
||||
use lemmy_db_views::structs::{LocalUserView, SiteView};
|
||||
|
@ -54,12 +60,17 @@ pub async fn update_site(
|
|||
SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?;
|
||||
}
|
||||
|
||||
let slur_regex = local_site_to_slur_regex(&local_site);
|
||||
let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
|
||||
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
|
||||
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
|
||||
|
||||
let site_form = SiteUpdateForm {
|
||||
name: data.name.clone(),
|
||||
sidebar: diesel_option_overwrite(data.sidebar.clone()),
|
||||
sidebar: diesel_option_overwrite(sidebar),
|
||||
description: diesel_option_overwrite(data.description.clone()),
|
||||
icon: diesel_option_overwrite_to_url(&data.icon)?,
|
||||
banner: diesel_option_overwrite_to_url(&data.banner)?,
|
||||
icon,
|
||||
banner,
|
||||
updated: Some(Some(naive_now())),
|
||||
..Default::default()
|
||||
};
|
||||
|
|
|
@ -50,7 +50,5 @@ enum_delegate = "0.2.0"
|
|||
|
||||
[dev-dependencies]
|
||||
serial_test = { workspace = true }
|
||||
reqwest-middleware = { workspace = true }
|
||||
task-local-extensions = "0.1.4"
|
||||
assert-json-diff = "2.0.2"
|
||||
pretty_assertions = { workspace = true }
|
||||
|
|
|
@ -8,7 +8,7 @@ use crate::{
|
|||
},
|
||||
activity_lists::AnnouncableActivities,
|
||||
insert_received_activity,
|
||||
objects::{community::ApubCommunity, person::ApubPerson},
|
||||
objects::{community::ApubCommunity, person::ApubPerson, read_from_string_or_source_opt},
|
||||
protocol::{activities::community::update::UpdateCommunity, InCommunity},
|
||||
};
|
||||
use activitypub_federation::{
|
||||
|
@ -18,8 +18,13 @@ use activitypub_federation::{
|
|||
};
|
||||
use lemmy_api_common::context::LemmyContext;
|
||||
use lemmy_db_schema::{
|
||||
source::{activity::ActivitySendTargets, community::Community, person::Person},
|
||||
source::{
|
||||
activity::ActivitySendTargets,
|
||||
community::{Community, CommunityUpdateForm},
|
||||
person::Person,
|
||||
},
|
||||
traits::Crud,
|
||||
utils::naive_now,
|
||||
};
|
||||
use lemmy_utils::error::LemmyError;
|
||||
use url::Url;
|
||||
|
@ -85,7 +90,33 @@ impl ActivityHandler for UpdateCommunity {
|
|||
insert_received_activity(&self.id, context).await?;
|
||||
let community = self.community(context).await?;
|
||||
|
||||
let community_update_form = self.object.into_update_form();
|
||||
let community_update_form = CommunityUpdateForm {
|
||||
title: Some(self.object.name.unwrap_or(self.object.preferred_username)),
|
||||
description: Some(read_from_string_or_source_opt(
|
||||
&self.object.summary,
|
||||
&None,
|
||||
&self.object.source,
|
||||
)),
|
||||
removed: None,
|
||||
published: self.object.published.map(Into::into),
|
||||
updated: Some(self.object.updated.map(Into::into)),
|
||||
deleted: None,
|
||||
nsfw: Some(self.object.sensitive.unwrap_or(false)),
|
||||
actor_id: Some(self.object.id.into()),
|
||||
local: None,
|
||||
private_key: None,
|
||||
hidden: None,
|
||||
public_key: Some(self.object.public_key.public_key_pem),
|
||||
last_refreshed_at: Some(naive_now()),
|
||||
icon: Some(self.object.icon.map(|i| i.url.into())),
|
||||
banner: Some(self.object.image.map(|i| i.url.into())),
|
||||
followers_url: Some(self.object.followers.into()),
|
||||
inbox_url: Some(self.object.inbox.into()),
|
||||
shared_inbox_url: Some(self.object.endpoints.map(|e| e.shared_inbox.into())),
|
||||
moderators_url: self.object.attributed_to.map(Into::into),
|
||||
posting_restricted_to_mods: self.object.posting_restricted_to_mods,
|
||||
featured_url: self.object.featured.map(Into::into),
|
||||
};
|
||||
|
||||
Community::update(&mut context.pool(), community.id, &community_update_form).await?;
|
||||
Ok(())
|
||||
|
|
|
@ -298,10 +298,7 @@ pub async fn import_settings(
|
|||
mod tests {
|
||||
#![allow(clippy::indexing_slicing)]
|
||||
|
||||
use crate::{
|
||||
api::user_settings_backup::{export_settings, import_settings},
|
||||
objects::tests::init_context,
|
||||
};
|
||||
use crate::api::user_settings_backup::{export_settings, import_settings};
|
||||
use activitypub_federation::config::Data;
|
||||
use lemmy_api_common::context::LemmyContext;
|
||||
use lemmy_db_schema::{
|
||||
|
@ -348,7 +345,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_settings_export_import() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
|
||||
let export_user =
|
||||
create_user("hanna".to_string(), Some("my bio".to_string()), &context).await?;
|
||||
|
@ -397,7 +394,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disallow_large_backup() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
|
||||
let export_user =
|
||||
create_user("hanna".to_string(), Some("my bio".to_string()), &context).await?;
|
||||
|
|
|
@ -106,11 +106,7 @@ mod tests {
|
|||
|
||||
use super::*;
|
||||
use crate::{
|
||||
objects::{
|
||||
community::tests::parse_lemmy_community,
|
||||
person::tests::parse_lemmy_person,
|
||||
tests::init_context,
|
||||
},
|
||||
objects::{community::tests::parse_lemmy_community, person::tests::parse_lemmy_person},
|
||||
protocol::tests::file_to_json_object,
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
|
@ -129,7 +125,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_lemmy_community_moderators() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let (new_mod, site) = parse_lemmy_person(&context).await?;
|
||||
let community = parse_lemmy_community(&context).await?;
|
||||
let community_id = community.id;
|
||||
|
|
|
@ -16,7 +16,10 @@ use activitypub_federation::{
|
|||
traits::Object,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex};
|
||||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
utils::{local_site_opt_to_slur_regex, process_markdown},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
comment::{Comment, CommentInsertForm, CommentUpdateForm},
|
||||
|
@ -29,7 +32,7 @@ use lemmy_db_schema::{
|
|||
};
|
||||
use lemmy_utils::{
|
||||
error::{LemmyError, LemmyErrorType},
|
||||
utils::{markdown::markdown_to_html, slurs::remove_slurs},
|
||||
utils::markdown::markdown_to_html,
|
||||
};
|
||||
use std::ops::Deref;
|
||||
use url::Url;
|
||||
|
@ -158,7 +161,7 @@ impl Object for ApubComment {
|
|||
|
||||
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
||||
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
|
||||
let content = remove_slurs(&content, slur_regex);
|
||||
let content = process_markdown(&content, slur_regex, context).await?;
|
||||
let language_id =
|
||||
LanguageTag::to_language_id_single(note.language, &mut context.pool()).await?;
|
||||
|
||||
|
@ -190,7 +193,6 @@ pub(crate) mod tests {
|
|||
instance::ApubSite,
|
||||
person::{tests::parse_lemmy_person, ApubPerson},
|
||||
post::ApubPost,
|
||||
tests::init_context,
|
||||
},
|
||||
protocol::tests::file_to_json_object,
|
||||
};
|
||||
|
@ -230,7 +232,7 @@ pub(crate) mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
pub(crate) async fn test_parse_lemmy_comment() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let url = Url::parse("https://enterprise.lemmy.ml/comment/38741")?;
|
||||
let data = prepare_comment_test(&url, &context).await?;
|
||||
|
||||
|
@ -255,7 +257,7 @@ pub(crate) mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_pleroma_comment() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let url = Url::parse("https://enterprise.lemmy.ml/comment/38741")?;
|
||||
let data = prepare_comment_test(&url, &context).await?;
|
||||
|
||||
|
|
|
@ -2,7 +2,7 @@ use crate::{
|
|||
activities::GetActorType,
|
||||
check_apub_id_valid,
|
||||
local_site_data_cached,
|
||||
objects::instance::fetch_instance_actor_for_object,
|
||||
objects::{instance::fetch_instance_actor_for_object, read_from_string_or_source_opt},
|
||||
protocol::{
|
||||
objects::{group::Group, Endpoints, LanguageTag},
|
||||
ImageObject,
|
||||
|
@ -17,15 +17,24 @@ use activitypub_federation::{
|
|||
use chrono::{DateTime, Utc};
|
||||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
utils::{generate_featured_url, generate_moderators_url, generate_outbox_url},
|
||||
utils::{
|
||||
generate_featured_url,
|
||||
generate_moderators_url,
|
||||
generate_outbox_url,
|
||||
local_site_opt_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_apub,
|
||||
},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
activity::ActorType,
|
||||
actor_language::CommunityLanguage,
|
||||
community::{Community, CommunityUpdateForm},
|
||||
community::{Community, CommunityInsertForm, CommunityUpdateForm},
|
||||
local_site::LocalSite,
|
||||
},
|
||||
traits::{ApubActor, Crud},
|
||||
utils::naive_now,
|
||||
};
|
||||
use lemmy_db_views_actor::structs::CommunityFollowerView;
|
||||
use lemmy_utils::{error::LemmyError, spawn_try_task, utils::markdown::markdown_to_html};
|
||||
|
@ -130,7 +139,38 @@ impl Object for ApubCommunity {
|
|||
) -> Result<ApubCommunity, LemmyError> {
|
||||
let instance_id = fetch_instance_actor_for_object(&group.id, context).await?;
|
||||
|
||||
let form = Group::into_insert_form(group.clone(), instance_id);
|
||||
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
||||
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
|
||||
let description = read_from_string_or_source_opt(&group.summary, &None, &group.source);
|
||||
let description = process_markdown_opt(&description, slur_regex, context).await?;
|
||||
let icon = proxy_image_link_opt_apub(group.icon.map(|i| i.url), context).await?;
|
||||
let banner = proxy_image_link_opt_apub(group.image.map(|i| i.url), context).await?;
|
||||
|
||||
let form = CommunityInsertForm {
|
||||
name: group.preferred_username.clone(),
|
||||
title: group.name.unwrap_or(group.preferred_username.clone()),
|
||||
description,
|
||||
removed: None,
|
||||
published: group.published,
|
||||
updated: group.updated,
|
||||
deleted: Some(false),
|
||||
nsfw: Some(group.sensitive.unwrap_or(false)),
|
||||
actor_id: Some(group.id.into()),
|
||||
local: Some(false),
|
||||
private_key: None,
|
||||
hidden: None,
|
||||
public_key: group.public_key.public_key_pem,
|
||||
last_refreshed_at: Some(naive_now()),
|
||||
icon,
|
||||
banner,
|
||||
followers_url: Some(group.followers.clone().into()),
|
||||
inbox_url: Some(group.inbox.into()),
|
||||
shared_inbox_url: group.endpoints.map(|e| e.shared_inbox.into()),
|
||||
moderators_url: group.attributed_to.clone().map(Into::into),
|
||||
posting_restricted_to_mods: group.posting_restricted_to_mods,
|
||||
instance_id,
|
||||
featured_url: group.featured.map(Into::into),
|
||||
};
|
||||
let languages =
|
||||
LanguageTag::to_language_id_multiple(group.language, &mut context.pool()).await?;
|
||||
|
||||
|
@ -212,7 +252,7 @@ impl ApubCommunity {
|
|||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
objects::{instance::tests::parse_lemmy_instance, tests::init_context},
|
||||
objects::instance::tests::parse_lemmy_instance,
|
||||
protocol::tests::file_to_json_object,
|
||||
};
|
||||
use activitypub_federation::fetch::collection_id::CollectionId;
|
||||
|
@ -241,7 +281,7 @@ pub(crate) mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_lemmy_community() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let site = parse_lemmy_instance(&context).await?;
|
||||
let community = parse_lemmy_community(&context).await?;
|
||||
|
||||
|
|
|
@ -17,13 +17,17 @@ use activitypub_federation::{
|
|||
traits::{Actor, Object},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex};
|
||||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
utils::{local_site_opt_to_slur_regex, process_markdown_opt, proxy_image_link_opt_apub},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
newtypes::InstanceId,
|
||||
source::{
|
||||
activity::ActorType,
|
||||
actor_language::SiteLanguage,
|
||||
instance::Instance as DbInstance,
|
||||
local_site::LocalSite,
|
||||
site::{Site, SiteInsertForm},
|
||||
},
|
||||
traits::Crud,
|
||||
|
@ -126,18 +130,23 @@ impl Object for ApubSite {
|
|||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
async fn from_json(apub: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, LemmyError> {
|
||||
async fn from_json(apub: Self::Kind, context: &Data<Self::DataType>) -> Result<Self, LemmyError> {
|
||||
let domain = apub.id.inner().domain().expect("group id has domain");
|
||||
let instance = DbInstance::read_or_create(&mut data.pool(), domain.to_string()).await?;
|
||||
let instance = DbInstance::read_or_create(&mut context.pool(), domain.to_string()).await?;
|
||||
|
||||
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
||||
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
|
||||
let sidebar = read_from_string_or_source_opt(&apub.content, &None, &apub.source);
|
||||
let sidebar = process_markdown_opt(&sidebar, slur_regex, context).await?;
|
||||
let icon = proxy_image_link_opt_apub(apub.icon.map(|i| i.url), context).await?;
|
||||
let banner = proxy_image_link_opt_apub(apub.image.map(|i| i.url), context).await?;
|
||||
|
||||
let site_form = SiteInsertForm {
|
||||
name: apub.name.clone(),
|
||||
sidebar,
|
||||
updated: apub.updated,
|
||||
icon: apub.icon.clone().map(|i| i.url.into()),
|
||||
banner: apub.image.clone().map(|i| i.url.into()),
|
||||
icon,
|
||||
banner,
|
||||
description: apub.summary,
|
||||
actor_id: Some(apub.id.clone().into()),
|
||||
last_refreshed_at: Some(naive_now()),
|
||||
|
@ -146,10 +155,11 @@ impl Object for ApubSite {
|
|||
private_key: None,
|
||||
instance_id: instance.id,
|
||||
};
|
||||
let languages = LanguageTag::to_language_id_multiple(apub.language, &mut data.pool()).await?;
|
||||
let languages =
|
||||
LanguageTag::to_language_id_multiple(apub.language, &mut context.pool()).await?;
|
||||
|
||||
let site = Site::create(&mut data.pool(), &site_form).await?;
|
||||
SiteLanguage::update(&mut data.pool(), languages, &site).await?;
|
||||
let site = Site::create(&mut context.pool(), &site_form).await?;
|
||||
SiteLanguage::update(&mut context.pool(), languages, &site).await?;
|
||||
Ok(site.into())
|
||||
}
|
||||
}
|
||||
|
@ -205,7 +215,7 @@ pub(in crate::objects) async fn fetch_instance_actor_for_object<T: Into<Url> + C
|
|||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::{objects::tests::init_context, protocol::tests::file_to_json_object};
|
||||
use crate::protocol::tests::file_to_json_object;
|
||||
use lemmy_db_schema::traits::Crud;
|
||||
use lemmy_utils::error::LemmyResult;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
@ -223,7 +233,7 @@ pub(crate) mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_lemmy_instance() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let site = parse_lemmy_instance(&context).await?;
|
||||
|
||||
assert_eq!(site.name, "Enterprise");
|
||||
|
|
|
@ -51,54 +51,3 @@ pub(crate) fn verify_is_remote_object(id: &Url, settings: &Settings) -> Result<(
|
|||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use activitypub_federation::config::{Data, FederationConfig};
|
||||
use anyhow::anyhow;
|
||||
use lemmy_api_common::{context::LemmyContext, request::client_builder};
|
||||
use lemmy_db_schema::{source::secret::Secret, utils::build_db_pool_for_tests};
|
||||
use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell, settings::SETTINGS};
|
||||
use reqwest::{Request, Response};
|
||||
use reqwest_middleware::{ClientBuilder, Middleware, Next};
|
||||
use task_local_extensions::Extensions;
|
||||
|
||||
struct BlockedMiddleware;
|
||||
|
||||
/// A reqwest middleware which blocks all requests
|
||||
#[async_trait::async_trait]
|
||||
impl Middleware for BlockedMiddleware {
|
||||
async fn handle(
|
||||
&self,
|
||||
_req: Request,
|
||||
_extensions: &mut Extensions,
|
||||
_next: Next<'_>,
|
||||
) -> reqwest_middleware::Result<Response> {
|
||||
Err(anyhow!("Network requests not allowed").into())
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: would be nice if we didnt have to use a full context for tests.
|
||||
pub(crate) async fn init_context() -> LemmyResult<Data<LemmyContext>> {
|
||||
// call this to run migrations
|
||||
let pool = build_db_pool_for_tests().await;
|
||||
|
||||
let client = client_builder(&SETTINGS).build()?;
|
||||
|
||||
let client = ClientBuilder::new(client).with(BlockedMiddleware).build();
|
||||
let secret = Secret {
|
||||
id: 0,
|
||||
jwt_secret: String::new(),
|
||||
};
|
||||
|
||||
let rate_limit_cell = RateLimitCell::with_test_config();
|
||||
|
||||
let context = LemmyContext::create(pool, client, secret, rate_limit_cell.clone());
|
||||
let config = FederationConfig::builder()
|
||||
.domain("example.com")
|
||||
.app_data(context)
|
||||
.build()
|
||||
.await?;
|
||||
Ok(config.to_request_data())
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,11 +20,17 @@ use activitypub_federation::{
|
|||
use chrono::{DateTime, Utc};
|
||||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
utils::{generate_outbox_url, local_site_opt_to_slur_regex},
|
||||
utils::{
|
||||
generate_outbox_url,
|
||||
local_site_opt_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_apub,
|
||||
},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
activity::ActorType,
|
||||
local_site::LocalSite,
|
||||
person::{Person as DbPerson, PersonInsertForm, PersonUpdateForm},
|
||||
},
|
||||
traits::{ApubActor, Crud},
|
||||
|
@ -144,7 +150,12 @@ impl Object for ApubPerson {
|
|||
) -> Result<ApubPerson, LemmyError> {
|
||||
let instance_id = fetch_instance_actor_for_object(&person.id, context).await?;
|
||||
|
||||
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
||||
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
|
||||
let bio = read_from_string_or_source_opt(&person.summary, &None, &person.source);
|
||||
let bio = process_markdown_opt(&bio, slur_regex, context).await?;
|
||||
let avatar = proxy_image_link_opt_apub(person.icon.map(|i| i.url), context).await?;
|
||||
let banner = proxy_image_link_opt_apub(person.image.map(|i| i.url), context).await?;
|
||||
|
||||
// Some Mastodon users have `name: ""` (empty string), need to convert that to `None`
|
||||
// https://github.com/mastodon/mastodon/issues/25233
|
||||
|
@ -156,8 +167,8 @@ impl Object for ApubPerson {
|
|||
banned: None,
|
||||
ban_expires: None,
|
||||
deleted: Some(false),
|
||||
avatar: person.icon.map(|i| i.url.into()),
|
||||
banner: person.image.map(|i| i.url.into()),
|
||||
avatar,
|
||||
banner,
|
||||
published: person.published.map(Into::into),
|
||||
updated: person.updated.map(Into::into),
|
||||
actor_id: Some(person.id.into()),
|
||||
|
@ -210,10 +221,7 @@ impl GetActorType for ApubPerson {
|
|||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
objects::{
|
||||
instance::{tests::parse_lemmy_instance, ApubSite},
|
||||
tests::init_context,
|
||||
},
|
||||
objects::instance::{tests::parse_lemmy_instance, ApubSite},
|
||||
protocol::{objects::instance::Instance, tests::file_to_json_object},
|
||||
};
|
||||
use activitypub_federation::fetch::object_id::ObjectId;
|
||||
|
@ -237,7 +245,7 @@ pub(crate) mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_lemmy_person() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let (person, site) = parse_lemmy_person(&context).await?;
|
||||
|
||||
assert_eq!(person.display_name, Some("Jean-Luc Picard".to_string()));
|
||||
|
@ -251,7 +259,7 @@ pub(crate) mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_pleroma_person() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
|
||||
// create and parse a fake pleroma instance actor, to avoid network request during test
|
||||
let mut json: Instance = file_to_json_object("assets/lemmy/objects/instance.json")?;
|
||||
|
|
|
@ -24,8 +24,14 @@ use chrono::{DateTime, Utc};
|
|||
use html2text::{from_read_with_decorator, render::text_renderer::TrivialDecorator};
|
||||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
request::fetch_site_data,
|
||||
utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex},
|
||||
request::fetch_link_metadata_opt,
|
||||
utils::{
|
||||
is_mod_or_admin,
|
||||
local_site_opt_to_sensitive,
|
||||
local_site_opt_to_slur_regex,
|
||||
process_markdown_opt,
|
||||
proxy_image_link_opt_apub,
|
||||
},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
self,
|
||||
|
@ -40,11 +46,7 @@ use lemmy_db_schema::{
|
|||
};
|
||||
use lemmy_utils::{
|
||||
error::LemmyError,
|
||||
utils::{
|
||||
markdown::markdown_to_html,
|
||||
slurs::{check_slurs_opt, remove_slurs},
|
||||
validation::check_url_scheme,
|
||||
},
|
||||
utils::{markdown::markdown_to_html, slurs::check_slurs_opt, validation::check_url_scheme},
|
||||
};
|
||||
use std::ops::Deref;
|
||||
use stringreader::StringReader;
|
||||
|
@ -111,6 +113,13 @@ impl Object for ApubPost {
|
|||
let community = Community::read(&mut context.pool(), community_id).await?;
|
||||
let language = LanguageTag::new_single(self.language_id, &mut context.pool()).await?;
|
||||
|
||||
let attachment = self
|
||||
.url
|
||||
.clone()
|
||||
.map(|url| Attachment::new(url.into(), self.url_content_type.clone()))
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let page = Page {
|
||||
kind: PageType::Page,
|
||||
id: self.ap_id.clone().into(),
|
||||
|
@ -121,7 +130,7 @@ impl Object for ApubPost {
|
|||
content: self.body.as_ref().map(|b| markdown_to_html(b)),
|
||||
media_type: Some(MediaTypeMarkdownOrHtml::Html),
|
||||
source: self.body.clone().map(Source::new),
|
||||
attachment: self.url.clone().map(Attachment::new).into_iter().collect(),
|
||||
attachment,
|
||||
image: self.thumbnail_url.clone().map(ImageObject::new),
|
||||
comments_enabled: Some(!self.locked),
|
||||
sensitive: Some(self.nsfw),
|
||||
|
@ -210,33 +219,22 @@ impl Object for ApubPost {
|
|||
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
||||
let allow_sensitive = local_site_opt_to_sensitive(&local_site);
|
||||
let page_is_sensitive = page.sensitive.unwrap_or(false);
|
||||
let include_image = allow_sensitive || !page_is_sensitive;
|
||||
let allow_generate_thumbnail = allow_sensitive || !page_is_sensitive;
|
||||
let mut thumbnail_url = page.image.map(|i| i.url);
|
||||
let do_generate_thumbnail = thumbnail_url.is_none() && allow_generate_thumbnail;
|
||||
|
||||
// Only fetch metadata if the post has a url and was not seen previously. We dont want to
|
||||
// waste resources by fetching metadata for the same post multiple times.
|
||||
// Additionally, only fetch image if content is not sensitive or is allowed on local site.
|
||||
let (metadata_res, thumbnail) = match &url {
|
||||
Some(url) if old_post.is_err() => {
|
||||
fetch_site_data(
|
||||
context.client(),
|
||||
context.settings(),
|
||||
Some(url),
|
||||
include_image,
|
||||
)
|
||||
.await
|
||||
}
|
||||
_ => (None, None),
|
||||
};
|
||||
// If no image was included with metadata, use post image instead when available.
|
||||
let thumbnail_url = thumbnail.or_else(|| page.image.map(|i| i.url.into()));
|
||||
// Generate local thumbnail only if no thumbnail was federated and 'sensitive' attributes allow it.
|
||||
let metadata = fetch_link_metadata_opt(url.as_ref(), do_generate_thumbnail, context).await;
|
||||
if let Some(thumbnail_url_) = metadata.thumbnail {
|
||||
thumbnail_url = Some(thumbnail_url_.into());
|
||||
}
|
||||
let url = proxy_image_link_opt_apub(url, context).await?;
|
||||
let thumbnail_url = proxy_image_link_opt_apub(thumbnail_url, context).await?;
|
||||
|
||||
let (embed_title, embed_description, embed_video_url) = metadata_res
|
||||
.map(|u| (u.title, u.description, u.embed_video_url))
|
||||
.unwrap_or_default();
|
||||
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
|
||||
|
||||
let body = read_from_string_or_source_opt(&page.content, &page.media_type, &page.source)
|
||||
.map(|s| remove_slurs(&s, slur_regex));
|
||||
let body = read_from_string_or_source_opt(&page.content, &page.media_type, &page.source);
|
||||
let body = process_markdown_opt(&body, slur_regex, context).await?;
|
||||
let language_id =
|
||||
LanguageTag::to_language_id_single(page.language, &mut context.pool()).await?;
|
||||
|
||||
|
@ -252,15 +250,16 @@ impl Object for ApubPost {
|
|||
updated: page.updated.map(Into::into),
|
||||
deleted: Some(false),
|
||||
nsfw: page.sensitive,
|
||||
embed_title,
|
||||
embed_description,
|
||||
embed_video_url,
|
||||
embed_title: metadata.opengraph_data.title,
|
||||
embed_description: metadata.opengraph_data.description,
|
||||
embed_video_url: metadata.opengraph_data.embed_video_url,
|
||||
thumbnail_url,
|
||||
ap_id: Some(page.id.clone().into()),
|
||||
local: Some(false),
|
||||
language_id,
|
||||
featured_community: None,
|
||||
featured_local: None,
|
||||
url_content_type: metadata.content_type,
|
||||
}
|
||||
} else {
|
||||
// if is mod action, only update locked/stickied fields, nothing else
|
||||
|
@ -299,7 +298,6 @@ mod tests {
|
|||
instance::ApubSite,
|
||||
person::{tests::parse_lemmy_person, ApubPerson},
|
||||
post::ApubPost,
|
||||
tests::init_context,
|
||||
},
|
||||
protocol::tests::file_to_json_object,
|
||||
};
|
||||
|
@ -311,7 +309,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_lemmy_post() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let (person, site) = parse_lemmy_person(&context).await?;
|
||||
let community = parse_lemmy_community(&context).await?;
|
||||
|
||||
|
@ -335,7 +333,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_convert_mastodon_post_title() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let (person, site) = parse_lemmy_person(&context).await?;
|
||||
let community = parse_lemmy_community(&context).await?;
|
||||
|
||||
|
|
|
@ -12,9 +12,13 @@ use activitypub_federation::{
|
|||
traits::Object,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use lemmy_api_common::{context::LemmyContext, utils::check_person_block};
|
||||
use lemmy_api_common::{
|
||||
context::LemmyContext,
|
||||
utils::{check_person_block, local_site_opt_to_slur_regex, process_markdown},
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
source::{
|
||||
local_site::LocalSite,
|
||||
person::Person,
|
||||
private_message::{PrivateMessage, PrivateMessageInsertForm},
|
||||
},
|
||||
|
@ -121,7 +125,10 @@ impl Object for ApubPrivateMessage {
|
|||
let recipient = note.to[0].dereference(context).await?;
|
||||
check_person_block(creator.id, recipient.id, &mut context.pool()).await?;
|
||||
|
||||
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
||||
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
|
||||
let content = read_from_string_or_source(¬e.content, &None, ¬e.source);
|
||||
let content = process_markdown(&content, slur_regex, context).await?;
|
||||
|
||||
let form = PrivateMessageInsertForm {
|
||||
creator_id: creator.id,
|
||||
|
@ -146,7 +153,6 @@ mod tests {
|
|||
objects::{
|
||||
instance::{tests::parse_lemmy_instance, ApubSite},
|
||||
person::ApubPerson,
|
||||
tests::init_context,
|
||||
},
|
||||
protocol::tests::file_to_json_object,
|
||||
};
|
||||
|
@ -185,7 +191,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_lemmy_pm() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let url = Url::parse("https://enterprise.lemmy.ml/private_message/1621")?;
|
||||
let data = prepare_comment_test(&url, &context).await?;
|
||||
let json: ChatMessage = file_to_json_object("assets/lemmy/objects/chat_message.json")?;
|
||||
|
@ -208,7 +214,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_parse_pleroma_pm() -> LemmyResult<()> {
|
||||
let context = init_context().await?;
|
||||
let context = LemmyContext::init_test_context().await;
|
||||
let url = Url::parse("https://enterprise.lemmy.ml/private_message/1621")?;
|
||||
let data = prepare_comment_test(&url, &context).await?;
|
||||
let pleroma_url = Url::parse("https://queer.hacktivis.me/objects/2")?;
|
||||
|
|
|
@ -25,11 +25,6 @@ use activitypub_federation::{
|
|||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex};
|
||||
use lemmy_db_schema::{
|
||||
newtypes::InstanceId,
|
||||
source::community::{CommunityInsertForm, CommunityUpdateForm},
|
||||
utils::naive_now,
|
||||
};
|
||||
use lemmy_utils::{
|
||||
error::LemmyError,
|
||||
utils::slurs::{check_slurs, check_slurs_opt},
|
||||
|
@ -94,64 +89,4 @@ impl Group {
|
|||
check_slurs_opt(&description, slur_regex)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn into_insert_form(self, instance_id: InstanceId) -> CommunityInsertForm {
|
||||
let description = read_from_string_or_source_opt(&self.summary, &None, &self.source);
|
||||
|
||||
CommunityInsertForm {
|
||||
name: self.preferred_username.clone(),
|
||||
title: self.name.unwrap_or(self.preferred_username.clone()),
|
||||
description,
|
||||
removed: None,
|
||||
published: self.published,
|
||||
updated: self.updated,
|
||||
deleted: Some(false),
|
||||
nsfw: Some(self.sensitive.unwrap_or(false)),
|
||||
actor_id: Some(self.id.into()),
|
||||
local: Some(false),
|
||||
private_key: None,
|
||||
hidden: None,
|
||||
public_key: self.public_key.public_key_pem,
|
||||
last_refreshed_at: Some(naive_now()),
|
||||
icon: self.icon.map(|i| i.url.into()),
|
||||
banner: self.image.map(|i| i.url.into()),
|
||||
followers_url: Some(self.followers.into()),
|
||||
inbox_url: Some(self.inbox.into()),
|
||||
shared_inbox_url: self.endpoints.map(|e| e.shared_inbox.into()),
|
||||
moderators_url: self.attributed_to.map(Into::into),
|
||||
posting_restricted_to_mods: self.posting_restricted_to_mods,
|
||||
instance_id,
|
||||
featured_url: self.featured.map(Into::into),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn into_update_form(self) -> CommunityUpdateForm {
|
||||
CommunityUpdateForm {
|
||||
title: Some(self.name.unwrap_or(self.preferred_username)),
|
||||
description: Some(read_from_string_or_source_opt(
|
||||
&self.summary,
|
||||
&None,
|
||||
&self.source,
|
||||
)),
|
||||
removed: None,
|
||||
published: self.published.map(Into::into),
|
||||
updated: Some(self.updated.map(Into::into)),
|
||||
deleted: None,
|
||||
nsfw: Some(self.sensitive.unwrap_or(false)),
|
||||
actor_id: Some(self.id.into()),
|
||||
local: None,
|
||||
private_key: None,
|
||||
hidden: None,
|
||||
public_key: Some(self.public_key.public_key_pem),
|
||||
last_refreshed_at: Some(naive_now()),
|
||||
icon: Some(self.icon.map(|i| i.url.into())),
|
||||
banner: Some(self.image.map(|i| i.url.into())),
|
||||
followers_url: Some(self.followers.into()),
|
||||
inbox_url: Some(self.inbox.into()),
|
||||
shared_inbox_url: Some(self.endpoints.map(|e| e.shared_inbox.into())),
|
||||
moderators_url: self.attributed_to.map(Into::into),
|
||||
posting_restricted_to_mods: self.posting_restricted_to_mods,
|
||||
featured_url: self.featured.map(Into::into),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,7 +20,6 @@ use activitypub_federation::{
|
|||
use chrono::{DateTime, Utc};
|
||||
use itertools::Itertools;
|
||||
use lemmy_api_common::context::LemmyContext;
|
||||
use lemmy_db_schema::newtypes::DbUrl;
|
||||
use lemmy_utils::error::{LemmyError, LemmyErrorType};
|
||||
use serde::{de::Error, Deserialize, Deserializer, Serialize};
|
||||
use serde_with::skip_serializing_none;
|
||||
|
@ -72,24 +71,25 @@ pub struct Page {
|
|||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct Link {
|
||||
pub(crate) href: Url,
|
||||
pub(crate) r#type: LinkType,
|
||||
href: Url,
|
||||
media_type: Option<String>,
|
||||
r#type: LinkType,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct Image {
|
||||
#[serde(rename = "type")]
|
||||
pub(crate) kind: ImageType,
|
||||
pub(crate) url: Url,
|
||||
kind: ImageType,
|
||||
url: Url,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct Document {
|
||||
#[serde(rename = "type")]
|
||||
pub(crate) kind: DocumentType,
|
||||
pub(crate) url: Url,
|
||||
kind: DocumentType,
|
||||
url: Url,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
|
@ -167,11 +167,21 @@ impl Page {
|
|||
}
|
||||
|
||||
impl Attachment {
|
||||
pub(crate) fn new(url: DbUrl) -> Attachment {
|
||||
Attachment::Link(Link {
|
||||
href: url.into(),
|
||||
r#type: Default::default(),
|
||||
})
|
||||
/// Creates new attachment for a given link and mime type.
|
||||
pub(crate) fn new(url: Url, media_type: Option<String>) -> Attachment {
|
||||
let is_image = media_type.clone().unwrap_or_default().starts_with("image");
|
||||
if is_image {
|
||||
Attachment::Image(Image {
|
||||
kind: Default::default(),
|
||||
url,
|
||||
})
|
||||
} else {
|
||||
Attachment::Link(Link {
|
||||
href: url,
|
||||
media_type,
|
||||
r#type: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1,35 +0,0 @@
|
|||
use crate::{
|
||||
newtypes::LocalUserId,
|
||||
schema::image_upload::dsl::{image_upload, local_user_id},
|
||||
source::image_upload::{ImageUpload, ImageUploadForm},
|
||||
utils::{get_conn, DbPool},
|
||||
};
|
||||
use diesel::{insert_into, result::Error, ExpressionMethods, QueryDsl, Table};
|
||||
use diesel_async::RunQueryDsl;
|
||||
|
||||
impl ImageUpload {
|
||||
pub async fn create(pool: &mut DbPool<'_>, form: &ImageUploadForm) -> Result<Self, Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
insert_into(image_upload)
|
||||
.values(form)
|
||||
.get_result::<Self>(conn)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_all_by_local_user_id(
|
||||
pool: &mut DbPool<'_>,
|
||||
user_id: &LocalUserId,
|
||||
) -> Result<Vec<Self>, Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
image_upload
|
||||
.filter(local_user_id.eq(user_id))
|
||||
.select(image_upload::all_columns())
|
||||
.load::<ImageUpload>(conn)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
diesel::delete(image_upload.find(alias)).execute(conn).await
|
||||
}
|
||||
}
|
78
crates/db_schema/src/impls/images.rs
Normal file
78
crates/db_schema/src/impls/images.rs
Normal file
|
@ -0,0 +1,78 @@
|
|||
use crate::{
|
||||
newtypes::{DbUrl, LocalUserId},
|
||||
schema::{
|
||||
local_image::dsl::{local_image, local_user_id, pictrs_alias},
|
||||
remote_image::dsl::{link, remote_image},
|
||||
},
|
||||
source::images::{LocalImage, LocalImageForm, RemoteImage, RemoteImageForm},
|
||||
utils::{get_conn, DbPool},
|
||||
};
|
||||
use diesel::{
|
||||
dsl::exists,
|
||||
insert_into,
|
||||
result::Error,
|
||||
select,
|
||||
ExpressionMethods,
|
||||
NotFound,
|
||||
QueryDsl,
|
||||
Table,
|
||||
};
|
||||
use diesel_async::RunQueryDsl;
|
||||
use url::Url;
|
||||
|
||||
impl LocalImage {
|
||||
pub async fn create(pool: &mut DbPool<'_>, form: &LocalImageForm) -> Result<Self, Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
insert_into(local_image)
|
||||
.values(form)
|
||||
.get_result::<Self>(conn)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_all_by_local_user_id(
|
||||
pool: &mut DbPool<'_>,
|
||||
user_id: &LocalUserId,
|
||||
) -> Result<Vec<Self>, Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
local_image
|
||||
.filter(local_user_id.eq(user_id))
|
||||
.select(local_image::all_columns())
|
||||
.load::<LocalImage>(conn)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
diesel::delete(local_image.filter(pictrs_alias.eq(alias)))
|
||||
.execute(conn)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteImage {
|
||||
pub async fn create(pool: &mut DbPool<'_>, links: Vec<Url>) -> Result<usize, Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
let forms = links
|
||||
.into_iter()
|
||||
.map(|url| RemoteImageForm { link: url.into() })
|
||||
.collect::<Vec<_>>();
|
||||
insert_into(remote_image)
|
||||
.values(forms)
|
||||
.on_conflict_do_nothing()
|
||||
.execute(conn)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn validate(pool: &mut DbPool<'_>, link_: DbUrl) -> Result<(), Error> {
|
||||
let conn = &mut get_conn(pool).await?;
|
||||
|
||||
let exists = select(exists(remote_image.filter((link).eq(link_))))
|
||||
.get_result::<bool>(conn)
|
||||
.await?;
|
||||
if exists {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(NotFound)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -11,7 +11,7 @@ pub mod email_verification;
|
|||
pub mod federation_allowlist;
|
||||
pub mod federation_blocklist;
|
||||
pub mod federation_queue_state;
|
||||
pub mod image_upload;
|
||||
pub mod images;
|
||||
pub mod instance;
|
||||
pub mod instance_block;
|
||||
pub mod language;
|
||||
|
|
|
@ -434,6 +434,7 @@ mod tests {
|
|||
language_id: Default::default(),
|
||||
featured_community: false,
|
||||
featured_local: false,
|
||||
url_content_type: None,
|
||||
};
|
||||
|
||||
// Post Like
|
||||
|
|
|
@ -301,15 +301,6 @@ diesel::table! {
|
|||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
image_upload (pictrs_alias) {
|
||||
local_user_id -> Int4,
|
||||
pictrs_alias -> Text,
|
||||
pictrs_delete_token -> Text,
|
||||
published -> Timestamptz,
|
||||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
instance (id) {
|
||||
id -> Int4,
|
||||
|
@ -341,6 +332,15 @@ diesel::table! {
|
|||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
local_image (pictrs_alias) {
|
||||
local_user_id -> Int4,
|
||||
pictrs_alias -> Text,
|
||||
pictrs_delete_token -> Text,
|
||||
published -> Timestamptz,
|
||||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
use diesel::sql_types::*;
|
||||
use super::sql_types::ListingTypeEnum;
|
||||
|
@ -692,6 +692,7 @@ diesel::table! {
|
|||
language_id -> Int4,
|
||||
featured_community -> Bool,
|
||||
featured_local -> Bool,
|
||||
url_content_type -> Nullable<Text>,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -807,6 +808,14 @@ diesel::table! {
|
|||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
remote_image (id) {
|
||||
id -> Int4,
|
||||
link -> Text,
|
||||
published -> Timestamptz,
|
||||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
secret (id) {
|
||||
id -> Int4,
|
||||
|
@ -922,9 +931,9 @@ diesel::joinable!(email_verification -> local_user (local_user_id));
|
|||
diesel::joinable!(federation_allowlist -> instance (instance_id));
|
||||
diesel::joinable!(federation_blocklist -> instance (instance_id));
|
||||
diesel::joinable!(federation_queue_state -> instance (instance_id));
|
||||
diesel::joinable!(image_upload -> local_user (local_user_id));
|
||||
diesel::joinable!(instance_block -> instance (instance_id));
|
||||
diesel::joinable!(instance_block -> person (person_id));
|
||||
diesel::joinable!(local_image -> local_user (local_user_id));
|
||||
diesel::joinable!(local_site -> site (site_id));
|
||||
diesel::joinable!(local_site_rate_limit -> local_site (local_site_id));
|
||||
diesel::joinable!(local_user -> person (person_id));
|
||||
|
@ -1002,10 +1011,10 @@ diesel::allow_tables_to_appear_in_same_query!(
|
|||
federation_allowlist,
|
||||
federation_blocklist,
|
||||
federation_queue_state,
|
||||
image_upload,
|
||||
instance,
|
||||
instance_block,
|
||||
language,
|
||||
local_image,
|
||||
local_site,
|
||||
local_site_rate_limit,
|
||||
local_user,
|
||||
|
@ -1040,6 +1049,7 @@ diesel::allow_tables_to_appear_in_same_query!(
|
|||
private_message_report,
|
||||
received_activity,
|
||||
registration_application,
|
||||
remote_image,
|
||||
secret,
|
||||
sent_activity,
|
||||
site,
|
||||
|
|
50
crates/db_schema/src/source/images.rs
Normal file
50
crates/db_schema/src/source/images.rs
Normal file
|
@ -0,0 +1,50 @@
|
|||
use crate::newtypes::{DbUrl, LocalUserId};
|
||||
#[cfg(feature = "full")]
|
||||
use crate::schema::{local_image, remote_image};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_with::skip_serializing_none;
|
||||
use std::fmt::Debug;
|
||||
use typed_builder::TypedBuilder;
|
||||
|
||||
#[skip_serializing_none]
|
||||
#[derive(PartialEq, Eq, Debug, Clone)]
|
||||
#[cfg_attr(feature = "full", derive(Queryable, Associations))]
|
||||
#[cfg_attr(feature = "full", diesel(table_name = local_image))]
|
||||
#[cfg_attr(
|
||||
feature = "full",
|
||||
diesel(belongs_to(crate::source::local_user::LocalUser))
|
||||
)]
|
||||
pub struct LocalImage {
|
||||
pub local_user_id: LocalUserId,
|
||||
pub pictrs_alias: String,
|
||||
pub pictrs_delete_token: String,
|
||||
pub published: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, TypedBuilder)]
|
||||
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
|
||||
#[cfg_attr(feature = "full", diesel(table_name = local_image))]
|
||||
pub struct LocalImageForm {
|
||||
pub local_user_id: LocalUserId,
|
||||
pub pictrs_alias: String,
|
||||
pub pictrs_delete_token: String,
|
||||
}
|
||||
|
||||
/// Stores all images which are hosted on remote domains. When attempting to proxy an image, it
|
||||
/// is checked against this table to avoid Lemmy being used as a general purpose proxy.
|
||||
#[skip_serializing_none]
|
||||
#[derive(PartialEq, Eq, Debug, Clone)]
|
||||
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
|
||||
#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
|
||||
pub struct RemoteImage {
|
||||
pub id: i32,
|
||||
pub link: DbUrl,
|
||||
pub published: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, TypedBuilder)]
|
||||
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
|
||||
#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
|
||||
pub struct RemoteImageForm {
|
||||
pub link: DbUrl,
|
||||
}
|
|
@ -16,7 +16,7 @@ pub mod email_verification;
|
|||
pub mod federation_allowlist;
|
||||
pub mod federation_blocklist;
|
||||
pub mod federation_queue_state;
|
||||
pub mod image_upload;
|
||||
pub mod images;
|
||||
pub mod instance;
|
||||
pub mod instance_block;
|
||||
pub mod language;
|
||||
|
|
|
@ -55,6 +55,7 @@ pub struct Post {
|
|||
pub featured_community: bool,
|
||||
/// Whether the post is featured to its site.
|
||||
pub featured_local: bool,
|
||||
pub url_content_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, TypedBuilder)]
|
||||
|
@ -85,6 +86,7 @@ pub struct PostInsertForm {
|
|||
pub language_id: Option<LanguageId>,
|
||||
pub featured_community: Option<bool>,
|
||||
pub featured_local: Option<bool>,
|
||||
pub url_content_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
|
@ -109,6 +111,7 @@ pub struct PostUpdateForm {
|
|||
pub language_id: Option<LanguageId>,
|
||||
pub featured_community: Option<bool>,
|
||||
pub featured_local: Option<bool>,
|
||||
pub url_content_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Debug)]
|
||||
|
|
|
@ -1015,6 +1015,7 @@ mod tests {
|
|||
language_id: Default::default(),
|
||||
featured_community: false,
|
||||
featured_local: false,
|
||||
url_content_type: None,
|
||||
},
|
||||
community: Community {
|
||||
id: data.inserted_community.id,
|
||||
|
|
|
@ -1468,6 +1468,7 @@ mod tests {
|
|||
language_id: LanguageId(47),
|
||||
featured_community: false,
|
||||
featured_local: false,
|
||||
url_content_type: None,
|
||||
},
|
||||
my_vote: None,
|
||||
unread_comments: 0,
|
||||
|
|
|
@ -33,4 +33,5 @@ url = { workspace = true }
|
|||
once_cell = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
urlencoding = { workspace = true }
|
||||
rss = "2.0.7"
|
||||
|
|
|
@ -6,6 +6,7 @@ use actix_web::{
|
|||
StatusCode,
|
||||
},
|
||||
web,
|
||||
web::Query,
|
||||
Error,
|
||||
HttpRequest,
|
||||
HttpResponse,
|
||||
|
@ -13,15 +14,17 @@ use actix_web::{
|
|||
use futures::stream::{Stream, StreamExt};
|
||||
use lemmy_api_common::context::LemmyContext;
|
||||
use lemmy_db_schema::source::{
|
||||
image_upload::{ImageUpload, ImageUploadForm},
|
||||
images::{LocalImage, LocalImageForm, RemoteImage},
|
||||
local_site::LocalSite,
|
||||
};
|
||||
use lemmy_db_views::structs::LocalUserView;
|
||||
use lemmy_utils::{rate_limit::RateLimitCell, REQWEST_TIMEOUT};
|
||||
use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell, REQWEST_TIMEOUT};
|
||||
use reqwest::Body;
|
||||
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use url::Url;
|
||||
use urlencoding::decode;
|
||||
|
||||
pub fn config(
|
||||
cfg: &mut web::ServiceConfig,
|
||||
|
@ -87,13 +90,14 @@ async fn upload(
|
|||
body: web::Payload,
|
||||
// require login
|
||||
local_user_view: LocalUserView,
|
||||
client: web::Data<ClientWithMiddleware>,
|
||||
context: web::Data<LemmyContext>,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
// TODO: check rate limit here
|
||||
let pictrs_config = context.settings().pictrs_config()?;
|
||||
let image_url = format!("{}image", pictrs_config.url);
|
||||
|
||||
let mut client_req = adapt_request(&req, context.client(), image_url);
|
||||
let mut client_req = adapt_request(&req, &client, image_url);
|
||||
|
||||
if let Some(addr) = req.head().peer_addr {
|
||||
client_req = client_req.header("X-Forwarded-For", addr.to_string())
|
||||
|
@ -109,12 +113,12 @@ async fn upload(
|
|||
let images = res.json::<Images>().await.map_err(error::ErrorBadRequest)?;
|
||||
if let Some(images) = &images.files {
|
||||
for uploaded_image in images {
|
||||
let form = ImageUploadForm {
|
||||
let form = LocalImageForm {
|
||||
local_user_id: local_user_view.local_user.id,
|
||||
pictrs_alias: uploaded_image.file.to_string(),
|
||||
pictrs_delete_token: uploaded_image.delete_token.to_string(),
|
||||
};
|
||||
ImageUpload::create(&mut context.pool(), &form)
|
||||
LocalImage::create(&mut context.pool(), &form)
|
||||
.await
|
||||
.map_err(error::ErrorBadRequest)?;
|
||||
}
|
||||
|
@ -158,15 +162,15 @@ async fn full_res(
|
|||
url
|
||||
};
|
||||
|
||||
image(url, req, client).await
|
||||
image(url, req, &client).await
|
||||
}
|
||||
|
||||
async fn image(
|
||||
url: String,
|
||||
req: HttpRequest,
|
||||
client: web::Data<ClientWithMiddleware>,
|
||||
client: &ClientWithMiddleware,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let mut client_req = adapt_request(&req, &client, url);
|
||||
let mut client_req = adapt_request(&req, client, url);
|
||||
|
||||
if let Some(addr) = req.head().peer_addr {
|
||||
client_req = client_req.header("X-Forwarded-For", addr.to_string());
|
||||
|
@ -212,13 +216,35 @@ async fn delete(
|
|||
|
||||
let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
|
||||
|
||||
ImageUpload::delete_by_alias(&mut context.pool(), &file)
|
||||
LocalImage::delete_by_alias(&mut context.pool(), &file)
|
||||
.await
|
||||
.map_err(error::ErrorBadRequest)?;
|
||||
|
||||
Ok(HttpResponse::build(res.status()).body(BodyStream::new(res.bytes_stream())))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ImageProxyParams {
|
||||
url: String,
|
||||
}
|
||||
|
||||
pub async fn image_proxy(
|
||||
Query(params): Query<ImageProxyParams>,
|
||||
context: web::Data<LemmyContext>,
|
||||
) -> LemmyResult<HttpResponse> {
|
||||
let url = Url::parse(&decode(¶ms.url)?)?;
|
||||
|
||||
// Check that url corresponds to a federated image so that this can't be abused as a proxy
|
||||
// for arbitrary purposes.
|
||||
RemoteImage::validate(&mut context.pool(), url.clone().into()).await?;
|
||||
|
||||
let pictrs_config = context.settings().pictrs_config()?;
|
||||
let url = format!("{}image/original?proxy={}", pictrs_config.url, ¶ms.url);
|
||||
let image_response = context.client().get(url).send().await?;
|
||||
|
||||
Ok(HttpResponse::Ok().streaming(image_response.bytes_stream()))
|
||||
}
|
||||
|
||||
fn make_send<S>(mut stream: S) -> impl Stream<Item = S::Item> + Send + Unpin + 'static
|
||||
where
|
||||
S: Stream + Unpin + 'static,
|
||||
|
|
|
@ -39,8 +39,8 @@ http = { workspace = true }
|
|||
doku = { workspace = true, features = ["url-2"] }
|
||||
uuid = { workspace = true, features = ["serde", "v4"] }
|
||||
rosetta-i18n = { workspace = true }
|
||||
percent-encoding = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
urlencoding = { workspace = true }
|
||||
openssl = "0.10.63"
|
||||
html2text = "0.6.0"
|
||||
deser-hjson = "2.2.4"
|
||||
|
|
|
@ -6,12 +6,13 @@ use crate::{
|
|||
use anyhow::{anyhow, Context};
|
||||
use deser_hjson::from_str;
|
||||
use once_cell::sync::Lazy;
|
||||
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
|
||||
use regex::Regex;
|
||||
use std::{env, fs, io::Error};
|
||||
use urlencoding::encode;
|
||||
|
||||
pub mod structs;
|
||||
|
||||
use crate::settings::structs::PictrsImageMode;
|
||||
use structs::DatabaseConnection;
|
||||
|
||||
static DEFAULT_CONFIG_FILE: &str = "config/config.hjson";
|
||||
|
@ -53,11 +54,11 @@ impl Settings {
|
|||
DatabaseConnection::Parts(parts) => {
|
||||
format!(
|
||||
"postgres://{}:{}@{}:{}/{}",
|
||||
utf8_percent_encode(&parts.user, NON_ALPHANUMERIC),
|
||||
utf8_percent_encode(&parts.password, NON_ALPHANUMERIC),
|
||||
encode(&parts.user),
|
||||
encode(&parts.password),
|
||||
parts.host,
|
||||
parts.port,
|
||||
utf8_percent_encode(&parts.database, NON_ALPHANUMERIC),
|
||||
encode(&parts.database),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
@ -112,3 +113,17 @@ impl Settings {
|
|||
.ok_or_else(|| anyhow!("images_disabled").into())
|
||||
}
|
||||
}
|
||||
|
||||
impl PictrsConfig {
|
||||
pub fn image_mode(&self) -> PictrsImageMode {
|
||||
if let Some(cache_external_link_previews) = self.cache_external_link_previews {
|
||||
if cache_external_link_previews {
|
||||
PictrsImageMode::StoreLinkPreviews
|
||||
} else {
|
||||
PictrsImageMode::None
|
||||
}
|
||||
} else {
|
||||
self.image_mode.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -12,7 +12,6 @@ pub struct Settings {
|
|||
/// settings related to the postgresql database
|
||||
#[default(Default::default())]
|
||||
pub database: DatabaseConfig,
|
||||
/// Settings related to activitypub federation
|
||||
/// Pictrs image server configuration.
|
||||
#[default(Some(Default::default()))]
|
||||
pub(crate) pictrs: Option<PictrsConfig>,
|
||||
|
@ -79,22 +78,43 @@ pub struct PictrsConfig {
|
|||
#[default(None)]
|
||||
pub api_key: Option<String>,
|
||||
|
||||
/// By default the thumbnails for external links are stored in pict-rs. This ensures that they
|
||||
/// can be reliably retrieved and can be resized using pict-rs APIs. However it also increases
|
||||
/// storage usage. In case this is disabled, the Opengraph image is directly returned as
|
||||
/// thumbnail.
|
||||
/// Backwards compatibility with 0.18.1. False is equivalent to `image_mode: None`, true is
|
||||
/// equivalent to `image_mode: StoreLinkPreviews`.
|
||||
///
|
||||
/// In some countries it is forbidden to copy preview images from newspaper articles and only
|
||||
/// hotlinking is allowed. If that is the case for your instance, make sure that this setting is
|
||||
/// disabled.
|
||||
#[default(true)]
|
||||
pub cache_external_link_previews: bool,
|
||||
/// To be removed in 0.20
|
||||
pub(super) cache_external_link_previews: Option<bool>,
|
||||
|
||||
/// Timeout for uploading images to pictrs (in seconds)
|
||||
/// Specifies how to handle remote images, so that users don't have to connect directly to remote servers.
|
||||
#[default(PictrsImageMode::StoreLinkPreviews)]
|
||||
pub(super) image_mode: PictrsImageMode,
|
||||
|
||||
/// Timeout for uploading images to pictrs (in seconds)
|
||||
#[default(30)]
|
||||
pub upload_timeout: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum PictrsImageMode {
|
||||
/// Leave images unchanged, don't generate any local thumbnails for post urls. Instead the the
|
||||
/// Opengraph image is directly returned as thumbnail
|
||||
None,
|
||||
/// Generate thumbnails for external post urls and store them persistently in pict-rs. This
|
||||
/// ensures that they can be reliably retrieved and can be resized using pict-rs APIs. However
|
||||
/// it also increases storage usage.
|
||||
///
|
||||
/// This is the default behaviour, and also matches Lemmy 0.18.
|
||||
#[default]
|
||||
StoreLinkPreviews,
|
||||
/// If enabled, all images from remote domains are rewritten to pass through `/api/v3/image_proxy`,
|
||||
/// including embedded images in markdown. Images are stored temporarily in pict-rs for caching.
|
||||
/// This improves privacy as users don't expose their IP to untrusted servers, and decreases load
|
||||
/// on other servers. However it increases bandwidth use for the local server.
|
||||
///
|
||||
/// Requires pict-rs 0.5
|
||||
ProxyAllImages,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)]
|
||||
#[serde(default)]
|
||||
pub struct DatabaseConfig {
|
||||
|
|
|
@ -1,113 +0,0 @@
|
|||
use markdown_it::MarkdownIt;
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
mod spoiler_rule;
|
||||
|
||||
static MARKDOWN_PARSER: Lazy<MarkdownIt> = Lazy::new(|| {
|
||||
let mut parser = MarkdownIt::new();
|
||||
markdown_it::plugins::cmark::add(&mut parser);
|
||||
markdown_it::plugins::extra::add(&mut parser);
|
||||
spoiler_rule::add(&mut parser);
|
||||
|
||||
parser
|
||||
});
|
||||
|
||||
/// Replace special HTML characters in API parameters to prevent XSS attacks.
|
||||
///
|
||||
/// Taken from https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.md#output-encoding-for-html-contexts
|
||||
///
|
||||
/// `>` is left in place because it is interpreted as markdown quote.
|
||||
pub fn sanitize_html(text: &str) -> String {
|
||||
text
|
||||
.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('\"', """)
|
||||
.replace('\'', "'")
|
||||
}
|
||||
|
||||
/// Converts text from markdown to HTML, while escaping special characters.
|
||||
pub fn markdown_to_html(text: &str) -> String {
|
||||
MARKDOWN_PARSER.parse(text).xrender()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used)]
|
||||
#![allow(clippy::indexing_slicing)]
|
||||
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn test_basic_markdown() {
|
||||
let tests: Vec<_> = vec![
|
||||
(
|
||||
"headings",
|
||||
"# h1\n## h2\n### h3\n#### h4\n##### h5\n###### h6",
|
||||
"<h1>h1</h1>\n<h2>h2</h2>\n<h3>h3</h3>\n<h4>h4</h4>\n<h5>h5</h5>\n<h6>h6</h6>\n"
|
||||
),
|
||||
(
|
||||
"line breaks",
|
||||
"First\rSecond",
|
||||
"<p>First\nSecond</p>\n"),
|
||||
(
|
||||
"emphasis",
|
||||
"__bold__ **bold** *italic* ***bold+italic***",
|
||||
"<p><strong>bold</strong> <strong>bold</strong> <em>italic</em> <em><strong>bold+italic</strong></em></p>\n"
|
||||
),
|
||||
(
|
||||
"blockquotes",
|
||||
"> #### Hello\n > \n > - Hola\n > - 안영 \n>> Goodbye\n",
|
||||
"<blockquote>\n<h4>Hello</h4>\n<ul>\n<li>Hola</li>\n<li>안영</li>\n</ul>\n<blockquote>\n<p>Goodbye</p>\n</blockquote>\n</blockquote>\n"
|
||||
),
|
||||
(
|
||||
"lists (ordered, unordered)",
|
||||
"1. pen\n2. apple\n3. apple pen\n- pen\n- pineapple\n- pineapple pen",
|
||||
"<ol>\n<li>pen</li>\n<li>apple</li>\n<li>apple pen</li>\n</ol>\n<ul>\n<li>pen</li>\n<li>pineapple</li>\n<li>pineapple pen</li>\n</ul>\n"
|
||||
),
|
||||
(
|
||||
"code and code blocks",
|
||||
"this is my amazing `code snippet` and my amazing ```code block```",
|
||||
"<p>this is my amazing <code>code snippet</code> and my amazing <code>code block</code></p>\n"
|
||||
),
|
||||
(
|
||||
"links",
|
||||
"[Lemmy](https://join-lemmy.org/ \"Join Lemmy!\")",
|
||||
"<p><a href=\"https://join-lemmy.org/\" title=\"Join Lemmy!\">Lemmy</a></p>\n"
|
||||
),
|
||||
(
|
||||
"images",
|
||||
"![My linked image](https://image.com \"image alt text\")",
|
||||
"<p><img src=\"https://image.com\" alt=\"My linked image\" title=\"image alt text\" /></p>\n"
|
||||
),
|
||||
// Ensure any custom plugins are added to 'MARKDOWN_PARSER' implementation.
|
||||
(
|
||||
"basic spoiler",
|
||||
"::: spoiler click to see more\nhow spicy!\n:::\n",
|
||||
"<details><summary>click to see more</summary><p>how spicy!\n</p></details>\n"
|
||||
),
|
||||
(
|
||||
"escape html special chars",
|
||||
"<script>alert('xss');</script> hello &\"",
|
||||
"<p><script>alert(‘xss’);</script> hello &"</p>\n"
|
||||
)
|
||||
];
|
||||
|
||||
tests.iter().for_each(|&(msg, input, expected)| {
|
||||
let result = markdown_to_html(input);
|
||||
|
||||
assert_eq!(
|
||||
result, expected,
|
||||
"Testing {}, with original input '{}'",
|
||||
msg, input
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_html() {
|
||||
let sanitized = sanitize_html("<script>alert('xss');</script> hello &\"'");
|
||||
let expected = "<script>alert('xss');</script> hello &"'";
|
||||
assert_eq!(expected, sanitized)
|
||||
}
|
||||
}
|
38
crates/utils/src/utils/markdown/link_rule.rs
Normal file
38
crates/utils/src/utils/markdown/link_rule.rs
Normal file
|
@ -0,0 +1,38 @@
|
|||
use markdown_it::{generics::inline::full_link, MarkdownIt, Node, NodeValue, Renderer};
|
||||
|
||||
/// Renders markdown links. Copied directly from markdown-it source, unlike original code it also
|
||||
/// sets `rel=nofollow` attribute.
|
||||
///
|
||||
/// TODO: We can set nofollow only if post was not made by mod/admin, but then we have to construct
|
||||
/// new parser for every invocation which might have performance implications.
|
||||
/// https://github.com/markdown-it-rust/markdown-it/blob/master/src/plugins/cmark/inline/link.rs
|
||||
#[derive(Debug)]
|
||||
pub struct Link {
|
||||
pub url: String,
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
impl NodeValue for Link {
|
||||
fn render(&self, node: &Node, fmt: &mut dyn Renderer) {
|
||||
let mut attrs = node.attrs.clone();
|
||||
attrs.push(("href", self.url.clone()));
|
||||
attrs.push(("rel", "nofollow".to_string()));
|
||||
|
||||
if let Some(title) = &self.title {
|
||||
attrs.push(("title", title.clone()));
|
||||
}
|
||||
|
||||
fmt.open("a", &attrs);
|
||||
fmt.contents(&node.children);
|
||||
fmt.close("a");
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add(md: &mut MarkdownIt) {
|
||||
full_link::add::<false>(md, |href, title| {
|
||||
Node::new(Link {
|
||||
url: href.unwrap_or_default(),
|
||||
title,
|
||||
})
|
||||
});
|
||||
}
|
246
crates/utils/src/utils/markdown/mod.rs
Normal file
246
crates/utils/src/utils/markdown/mod.rs
Normal file
|
@ -0,0 +1,246 @@
|
|||
use crate::settings::SETTINGS;
|
||||
use markdown_it::{plugins::cmark::inline::image::Image, MarkdownIt};
|
||||
use once_cell::sync::Lazy;
|
||||
use url::Url;
|
||||
use urlencoding::encode;
|
||||
|
||||
mod link_rule;
|
||||
mod spoiler_rule;
|
||||
|
||||
static MARKDOWN_PARSER: Lazy<MarkdownIt> = Lazy::new(|| {
|
||||
let mut parser = MarkdownIt::new();
|
||||
markdown_it::plugins::cmark::add(&mut parser);
|
||||
markdown_it::plugins::extra::add(&mut parser);
|
||||
spoiler_rule::add(&mut parser);
|
||||
link_rule::add(&mut parser);
|
||||
|
||||
parser
|
||||
});
|
||||
|
||||
/// Replace special HTML characters in API parameters to prevent XSS attacks.
|
||||
///
|
||||
/// Taken from https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.md#output-encoding-for-html-contexts
|
||||
///
|
||||
/// `>` is left in place because it is interpreted as markdown quote.
|
||||
pub fn sanitize_html(text: &str) -> String {
|
||||
text
|
||||
.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('\"', """)
|
||||
.replace('\'', "'")
|
||||
}
|
||||
|
||||
pub fn markdown_to_html(text: &str) -> String {
|
||||
MARKDOWN_PARSER.parse(text).xrender()
|
||||
}
|
||||
|
||||
/// Rewrites all links to remote domains in markdown, so they go through `/api/v3/image_proxy`.
|
||||
pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec<Url>) {
|
||||
let ast = MARKDOWN_PARSER.parse(&src);
|
||||
let mut links_offsets = vec![];
|
||||
|
||||
// Walk the syntax tree to find positions of image links
|
||||
ast.walk(|node, _depth| {
|
||||
if let Some(image) = node.cast::<Image>() {
|
||||
// srcmap is always present for image
|
||||
// https://github.com/markdown-it-rust/markdown-it/issues/36#issuecomment-1777844387
|
||||
let node_offsets = node.srcmap.expect("srcmap is none").get_byte_offsets();
|
||||
// necessary for custom emojis which look like `![name](url "title")`
|
||||
let start_offset = node_offsets.1
|
||||
- image.url.len()
|
||||
- 1
|
||||
- image
|
||||
.title
|
||||
.as_ref()
|
||||
.map(|t| t.len() + 3)
|
||||
.unwrap_or_default();
|
||||
let end_offset = node_offsets.1 - 1;
|
||||
|
||||
links_offsets.push((start_offset, end_offset));
|
||||
}
|
||||
});
|
||||
|
||||
let mut links = vec![];
|
||||
// Go through the collected links in reverse order
|
||||
while let Some((start, end)) = links_offsets.pop() {
|
||||
let content = src.get(start..end).unwrap_or_default();
|
||||
// necessary for custom emojis which look like `![name](url "title")`
|
||||
let (url, extra) = if content.contains(' ') {
|
||||
let split = content.split_once(' ').expect("split is valid");
|
||||
(split.0, Some(split.1))
|
||||
} else {
|
||||
(content, None)
|
||||
};
|
||||
match Url::parse(url) {
|
||||
Ok(parsed) => {
|
||||
links.push(parsed.clone());
|
||||
// If link points to remote domain, replace with proxied link
|
||||
if parsed.domain() != Some(&SETTINGS.hostname) {
|
||||
let mut proxied = format!(
|
||||
"{}/api/v3/image_proxy?url={}",
|
||||
SETTINGS.get_protocol_and_hostname(),
|
||||
encode(url),
|
||||
);
|
||||
// restore custom emoji format
|
||||
if let Some(extra) = extra {
|
||||
proxied = format!("{proxied} {extra}");
|
||||
}
|
||||
src.replace_range(start..end, &proxied);
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// If its not a valid url, replace with empty text
|
||||
src.replace_range(start..end, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(src, links)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::unwrap_used)]
|
||||
#![allow(clippy::indexing_slicing)]
|
||||
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn test_basic_markdown() {
|
||||
let tests: Vec<_> = vec![
|
||||
(
|
||||
"headings",
|
||||
"# h1\n## h2\n### h3\n#### h4\n##### h5\n###### h6",
|
||||
"<h1>h1</h1>\n<h2>h2</h2>\n<h3>h3</h3>\n<h4>h4</h4>\n<h5>h5</h5>\n<h6>h6</h6>\n"
|
||||
),
|
||||
(
|
||||
"line breaks",
|
||||
"First\rSecond",
|
||||
"<p>First\nSecond</p>\n"),
|
||||
(
|
||||
"emphasis",
|
||||
"__bold__ **bold** *italic* ***bold+italic***",
|
||||
"<p><strong>bold</strong> <strong>bold</strong> <em>italic</em> <em><strong>bold+italic</strong></em></p>\n"
|
||||
),
|
||||
(
|
||||
"blockquotes",
|
||||
"> #### Hello\n > \n > - Hola\n > - 안영 \n>> Goodbye\n",
|
||||
"<blockquote>\n<h4>Hello</h4>\n<ul>\n<li>Hola</li>\n<li>안영</li>\n</ul>\n<blockquote>\n<p>Goodbye</p>\n</blockquote>\n</blockquote>\n"
|
||||
),
|
||||
(
|
||||
"lists (ordered, unordered)",
|
||||
"1. pen\n2. apple\n3. apple pen\n- pen\n- pineapple\n- pineapple pen",
|
||||
"<ol>\n<li>pen</li>\n<li>apple</li>\n<li>apple pen</li>\n</ol>\n<ul>\n<li>pen</li>\n<li>pineapple</li>\n<li>pineapple pen</li>\n</ul>\n"
|
||||
),
|
||||
(
|
||||
"code and code blocks",
|
||||
"this is my amazing `code snippet` and my amazing ```code block```",
|
||||
"<p>this is my amazing <code>code snippet</code> and my amazing <code>code block</code></p>\n"
|
||||
),
|
||||
// Links with added nofollow attribute
|
||||
(
|
||||
"links",
|
||||
"[Lemmy](https://join-lemmy.org/ \"Join Lemmy!\")",
|
||||
"<p><a href=\"https://join-lemmy.org/\" rel=\"nofollow\" title=\"Join Lemmy!\">Lemmy</a></p>\n"
|
||||
),
|
||||
// Remote images with proxy
|
||||
(
|
||||
"images",
|
||||
"![My linked image](https://example.com/image.png \"image alt text\")",
|
||||
"<p><img src=\"https://example.com/image.png\" alt=\"My linked image\" title=\"image alt text\" /></p>\n"
|
||||
),
|
||||
// Local images without proxy
|
||||
(
|
||||
"images",
|
||||
"![My linked image](https://lemmy-alpha/image.png \"image alt text\")",
|
||||
"<p><img src=\"https://lemmy-alpha/image.png\" alt=\"My linked image\" title=\"image alt text\" /></p>\n"
|
||||
),
|
||||
// Ensure spoiler plugin is added
|
||||
(
|
||||
"basic spoiler",
|
||||
"::: spoiler click to see more\nhow spicy!\n:::\n",
|
||||
"<details><summary>click to see more</summary><p>how spicy!\n</p></details>\n"
|
||||
),
|
||||
(
|
||||
"escape html special chars",
|
||||
"<script>alert('xss');</script> hello &\"",
|
||||
"<p><script>alert(‘xss’);</script> hello &"</p>\n"
|
||||
)
|
||||
];
|
||||
|
||||
tests.iter().for_each(|&(msg, input, expected)| {
|
||||
let result = markdown_to_html(input);
|
||||
|
||||
assert_eq!(
|
||||
result, expected,
|
||||
"Testing {}, with original input '{}'",
|
||||
msg, input
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_proxy_images() {
|
||||
let tests: Vec<_> =
|
||||
vec![
|
||||
(
|
||||
"remote image proxied",
|
||||
"![link](http://example.com/image.jpg)",
|
||||
"![link](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)",
|
||||
),
|
||||
(
|
||||
"local image unproxied",
|
||||
"![link](http://lemmy-alpha/image.jpg)",
|
||||
"![link](http://lemmy-alpha/image.jpg)",
|
||||
),
|
||||
(
|
||||
"multiple image links",
|
||||
"![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)",
|
||||
"![link](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)",
|
||||
),
|
||||
(
|
||||
"empty link handled",
|
||||
"![image]()",
|
||||
"![image]()"
|
||||
),
|
||||
(
|
||||
"empty label handled",
|
||||
"![](http://example.com/image.jpg)",
|
||||
"![](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
|
||||
),
|
||||
(
|
||||
"invalid image link removed",
|
||||
"![image](http-not-a-link)",
|
||||
"![image]()"
|
||||
),
|
||||
(
|
||||
"label with nested markdown handled",
|
||||
"![a *b* c](http://example.com/image.jpg)",
|
||||
"![a *b* c](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
|
||||
),
|
||||
(
|
||||
"custom emoji support",
|
||||
r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#,
|
||||
r#"![party-blob](https://lemmy-alpha/api/v3/image_proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#
|
||||
)
|
||||
];
|
||||
|
||||
tests.iter().for_each(|&(msg, input, expected)| {
|
||||
let result = markdown_rewrite_image_links(input.to_string());
|
||||
|
||||
assert_eq!(
|
||||
result.0, expected,
|
||||
"Testing {}, with original input '{}'",
|
||||
msg, input
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_html() {
|
||||
let sanitized = sanitize_html("<script>alert('xss');</script> hello &\"'");
|
||||
let expected = "<script>alert('xss');</script> hello &"'";
|
||||
assert_eq!(expected, sanitized)
|
||||
}
|
||||
}
|
|
@ -10,4 +10,8 @@
|
|||
database: {
|
||||
host: postgres_epsilon
|
||||
}
|
||||
pictrs: {
|
||||
api_key: "my-pictrs-key"
|
||||
image_mode: ProxyAllImages
|
||||
}
|
||||
}
|
||||
|
|
|
@ -10,4 +10,8 @@
|
|||
database: {
|
||||
host: postgres_gamma
|
||||
}
|
||||
pictrs: {
|
||||
api_key: "my-pictrs-key"
|
||||
image_mode: ProxyAllImages
|
||||
}
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@
|
|||
pictrs: {
|
||||
url: "http://pictrs:8080/"
|
||||
# api_key: "API_KEY"
|
||||
image_proxy: true
|
||||
cache_external_link_previews: true
|
||||
}
|
||||
|
||||
|
|
4
migrations/2023-10-24-131607_proxy_links/down.sql
Normal file
4
migrations/2023-10-24-131607_proxy_links/down.sql
Normal file
|
@ -0,0 +1,4 @@
|
|||
DROP TABLE remote_image;
|
||||
|
||||
ALTER TABLE local_image RENAME TO image_upload;
|
||||
|
8
migrations/2023-10-24-131607_proxy_links/up.sql
Normal file
8
migrations/2023-10-24-131607_proxy_links/up.sql
Normal file
|
@ -0,0 +1,8 @@
|
|||
CREATE TABLE remote_image (
|
||||
id serial PRIMARY KEY,
|
||||
link text NOT NULL UNIQUE,
|
||||
published timestamptz DEFAULT now() NOT NULL
|
||||
);
|
||||
|
||||
ALTER TABLE image_upload RENAME TO local_image;
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
ALTER TABLE post
|
||||
DROP COLUMN url_content_type;
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
ALTER TABLE post
|
||||
ADD COLUMN url_content_type text;
|
||||
|
|
@ -130,11 +130,13 @@ use lemmy_apub::api::{
|
|||
search::search,
|
||||
user_settings_backup::{export_settings, import_settings},
|
||||
};
|
||||
use lemmy_routes::images::image_proxy;
|
||||
use lemmy_utils::rate_limit::RateLimitCell;
|
||||
|
||||
pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) {
|
||||
cfg.service(
|
||||
web::scope("/api/v3")
|
||||
.route("/image_proxy", web::get().to(image_proxy))
|
||||
// Site
|
||||
.service(
|
||||
web::scope("/site")
|
||||
|
|
13
src/lib.rs
13
src/lib.rs
|
@ -53,7 +53,7 @@ use lemmy_utils::{
|
|||
};
|
||||
use prometheus::default_registry;
|
||||
use prometheus_metrics::serve_prometheus;
|
||||
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
|
||||
use reqwest_middleware::ClientBuilder;
|
||||
use reqwest_tracing::TracingMiddleware;
|
||||
use serde_json::json;
|
||||
use std::{env, ops::Deref};
|
||||
|
@ -198,15 +198,10 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
|
|||
startup_server_handle.stop(true).await;
|
||||
}
|
||||
|
||||
// Pictrs cannot use proxy
|
||||
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
|
||||
.with(TracingMiddleware::default())
|
||||
.build();
|
||||
Some(create_http_server(
|
||||
federation_config.clone(),
|
||||
SETTINGS.clone(),
|
||||
federation_enabled,
|
||||
pictrs_client,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
|
@ -272,7 +267,6 @@ fn create_http_server(
|
|||
federation_config: FederationConfig<LemmyContext>,
|
||||
settings: Settings,
|
||||
federation_enabled: bool,
|
||||
pictrs_client: ClientWithMiddleware,
|
||||
) -> Result<ServerHandle, LemmyError> {
|
||||
// this must come before the HttpServer creation
|
||||
// creates a middleware that populates http metrics for each path, method, and status code
|
||||
|
@ -284,6 +278,11 @@ fn create_http_server(
|
|||
let context: LemmyContext = federation_config.deref().clone();
|
||||
let rate_limit_cell = federation_config.rate_limit_cell().clone();
|
||||
|
||||
// Pictrs cannot use proxy
|
||||
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
|
||||
.with(TracingMiddleware::default())
|
||||
.build();
|
||||
|
||||
// Create Http server
|
||||
let bind = (settings.bind, settings.port);
|
||||
let server = HttpServer::new(move || {
|
||||
|
|
Loading…
Reference in a new issue