Cache federation blocklist (#3486)

* Cache federation blocklist

* revert submodule change
This commit is contained in:
Nutomic 2023-07-05 17:08:02 +02:00 committed by GitHub
parent 1e99e8b9dc
commit ebaf69bd70
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 351 additions and 86 deletions

293
Cargo.lock generated
View file

@ -451,6 +451,35 @@ dependencies = [
"syn 1.0.103", "syn 1.0.103",
] ]
[[package]]
name = "async-io"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af"
dependencies = [
"async-lock",
"autocfg",
"cfg-if",
"concurrent-queue",
"futures-lite",
"log",
"parking",
"polling",
"rustix 0.37.22",
"slab",
"socket2 0.4.9",
"waker-fn",
]
[[package]]
name = "async-lock"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7"
dependencies = [
"event-listener",
]
[[package]] [[package]]
name = "async-stream" name = "async-stream"
version = "0.3.3" version = "0.3.3"
@ -774,6 +803,12 @@ version = "3.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
[[package]]
name = "bytecount"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c"
[[package]] [[package]]
name = "bytemuck" name = "bytemuck"
version = "1.12.1" version = "1.12.1"
@ -801,6 +836,15 @@ dependencies = [
"bytes", "bytes",
] ]
[[package]]
name = "camino"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c530edf18f37068ac2d977409ed5cd50d53d73bc653c7647b48eb78976ac9ae2"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "captcha" name = "captcha"
version = "0.0.9" version = "0.0.9"
@ -815,6 +859,28 @@ dependencies = [
"serde_json", "serde_json",
] ]
[[package]]
name = "cargo-platform"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27"
dependencies = [
"serde",
]
[[package]]
name = "cargo_metadata"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
dependencies = [
"camino",
"cargo-platform",
"semver",
"serde",
"serde_json",
]
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.73" version = "1.0.73"
@ -961,6 +1027,15 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "concurrent-queue"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "config" name = "config"
version = "0.13.3" version = "0.13.3"
@ -1751,6 +1826,17 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "errno"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys 0.48.0",
]
[[package]] [[package]]
name = "errno-dragonfly" name = "errno-dragonfly"
version = "0.1.2" version = "0.1.2"
@ -1761,6 +1847,21 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "error-chain"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
dependencies = [
"version_check",
]
[[package]]
name = "event-listener"
version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
[[package]] [[package]]
name = "eyre" name = "eyre"
version = "0.6.8" version = "0.6.8"
@ -1937,6 +2038,21 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
[[package]]
name = "futures-lite"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce"
dependencies = [
"fastrand",
"futures-core",
"futures-io",
"memchr",
"parking",
"pin-project-lite",
"waker-fn",
]
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.28" version = "0.3.28"
@ -2027,6 +2143,12 @@ version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793"
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.3.14" version = "0.3.14"
@ -2107,6 +2229,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "hermit-abi"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
[[package]] [[package]]
name = "hex" name = "hex"
version = "0.4.3" version = "0.4.3"
@ -2425,12 +2553,13 @@ dependencies = [
[[package]] [[package]]
name = "io-lifetimes" name = "io-lifetimes"
version = "1.0.3" version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
dependencies = [ dependencies = [
"hermit-abi 0.3.2",
"libc", "libc",
"windows-sys 0.42.0", "windows-sys 0.48.0",
] ]
[[package]] [[package]]
@ -2460,7 +2589,7 @@ checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189"
dependencies = [ dependencies = [
"hermit-abi 0.2.6", "hermit-abi 0.2.6",
"io-lifetimes", "io-lifetimes",
"rustix", "rustix 0.36.5",
"windows-sys 0.42.0", "windows-sys 0.42.0",
] ]
@ -2650,6 +2779,7 @@ dependencies = [
"lemmy_db_views", "lemmy_db_views",
"lemmy_db_views_actor", "lemmy_db_views_actor",
"lemmy_utils", "lemmy_utils",
"moka",
"once_cell", "once_cell",
"reqwest", "reqwest",
"reqwest-middleware", "reqwest-middleware",
@ -3007,6 +3137,12 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
[[package]]
name = "linux-raw-sys"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
[[package]] [[package]]
name = "local-channel" name = "local-channel"
version = "0.1.3" version = "0.1.3"
@ -3063,6 +3199,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mach2"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "markdown-it" name = "markdown-it"
version = "0.5.0" version = "0.5.0"
@ -3279,6 +3424,31 @@ dependencies = [
"windows-sys 0.36.1", "windows-sys 0.36.1",
] ]
[[package]]
name = "moka"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "206bf83f415b0579fd885fe0804eb828e727636657dc1bf73d80d2f1218e14a1"
dependencies = [
"async-io",
"async-lock",
"crossbeam-channel",
"crossbeam-epoch",
"crossbeam-utils",
"futures-util",
"once_cell",
"parking_lot 0.12.1",
"quanta",
"rustc_version",
"scheduled-thread-pool",
"skeptic",
"smallvec",
"tagptr",
"thiserror",
"triomphe",
"uuid",
]
[[package]] [[package]]
name = "multimap" name = "multimap"
version = "0.8.3" version = "0.8.3"
@ -3631,6 +3801,12 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
[[package]]
name = "parking"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e"
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.11.2" version = "0.11.2"
@ -3983,6 +4159,22 @@ dependencies = [
"miniz_oxide 0.5.4", "miniz_oxide 0.5.4",
] ]
[[package]]
name = "polling"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce"
dependencies = [
"autocfg",
"bitflags 1.3.2",
"cfg-if",
"concurrent-queue",
"libc",
"log",
"pin-project-lite",
"windows-sys 0.48.0",
]
[[package]] [[package]]
name = "postgres-protocol" name = "postgres-protocol"
version = "0.6.5" version = "0.6.5"
@ -4197,6 +4389,33 @@ dependencies = [
"cc", "cc",
] ]
[[package]]
name = "pulldown-cmark"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998"
dependencies = [
"bitflags 1.3.2",
"memchr",
"unicase",
]
[[package]]
name = "quanta"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab"
dependencies = [
"crossbeam-utils",
"libc",
"mach2",
"once_cell",
"raw-cpuid",
"wasi 0.11.0+wasi-snapshot-preview1",
"web-sys",
"winapi",
]
[[package]] [[package]]
name = "quick-xml" name = "quick-xml"
version = "0.27.1" version = "0.27.1"
@ -4320,6 +4539,15 @@ dependencies = [
"rand_core 0.5.1", "rand_core 0.5.1",
] ]
[[package]]
name = "raw-cpuid"
version = "10.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
dependencies = [
"bitflags 1.3.2",
]
[[package]] [[package]]
name = "readonly" name = "readonly"
version = "0.2.8" version = "0.2.8"
@ -4571,13 +4799,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588"
dependencies = [ dependencies = [
"bitflags 1.3.2", "bitflags 1.3.2",
"errno", "errno 0.2.8",
"io-lifetimes", "io-lifetimes",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys 0.1.4",
"windows-sys 0.42.0", "windows-sys 0.42.0",
] ]
[[package]]
name = "rustix"
version = "0.37.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8818fa822adcc98b18fedbb3632a6a33213c070556b5aa7c4c8cc21cff565c4c"
dependencies = [
"bitflags 1.3.2",
"errno 0.3.1",
"io-lifetimes",
"libc",
"linux-raw-sys 0.3.8",
"windows-sys 0.48.0",
]
[[package]] [[package]]
name = "rustls" name = "rustls"
version = "0.20.7" version = "0.20.7"
@ -4668,6 +4910,15 @@ dependencies = [
"windows-sys 0.36.1", "windows-sys 0.36.1",
] ]
[[package]]
name = "scheduled-thread-pool"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
dependencies = [
"parking_lot 0.12.1",
]
[[package]] [[package]]
name = "scoped-futures" name = "scoped-futures"
version = "0.1.3" version = "0.1.3"
@ -4745,6 +4996,9 @@ name = "semver"
version = "1.0.14" version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "serde" name = "serde"
@ -4924,6 +5178,21 @@ version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
[[package]]
name = "skeptic"
version = "0.13.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
dependencies = [
"bytecount",
"cargo_metadata",
"error-chain",
"glob",
"pulldown-cmark",
"tempfile",
"walkdir",
]
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.7" version = "0.4.7"
@ -5279,6 +5548,12 @@ dependencies = [
"yaml-rust", "yaml-rust",
] ]
[[package]]
name = "tagptr"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]] [[package]]
name = "tap" name = "tap"
version = "1.0.1" version = "1.0.1"
@ -6168,6 +6443,12 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]]
name = "waker-fn"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
[[package]] [[package]]
name = "walkdir" name = "walkdir"
version = "2.3.2" version = "2.3.2"

View file

@ -41,6 +41,7 @@ once_cell = { workspace = true }
html2md = "0.2.14" html2md = "0.2.14"
serde_with = { workspace = true } serde_with = { workspace = true }
enum_delegate = "0.2.0" enum_delegate = "0.2.0"
moka = { version = "0.11", features = ["future"] }
[dev-dependencies] [dev-dependencies]
serial_test = { workspace = true } serial_test = { workspace = true }

View file

@ -1,6 +1,7 @@
use crate::fetcher::post_or_comment::PostOrComment; use crate::fetcher::post_or_comment::PostOrComment;
use activitypub_federation::config::{Data, UrlVerifier}; use activitypub_federation::config::{Data, UrlVerifier};
use async_trait::async_trait; use async_trait::async_trait;
use futures::future::join3;
use lemmy_api_common::context::LemmyContext; use lemmy_api_common::context::LemmyContext;
use lemmy_db_schema::{ use lemmy_db_schema::{
source::{ source::{
@ -11,9 +12,11 @@ use lemmy_db_schema::{
traits::Crud, traits::Crud,
utils::DbPool, utils::DbPool,
}; };
use lemmy_utils::{error::LemmyError, settings::structs::Settings}; use lemmy_utils::error::{LemmyError, LemmyResult};
use moka::future::Cache;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use serde::Serialize; use serde::Serialize;
use std::{sync::Arc, time::Duration};
use url::Url; use url::Url;
pub mod activities; pub mod activities;
@ -27,6 +30,11 @@ pub mod objects;
pub mod protocol; pub mod protocol;
pub const FEDERATION_HTTP_FETCH_LIMIT: u32 = 50; pub const FEDERATION_HTTP_FETCH_LIMIT: u32 = 50;
/// All incoming and outgoing federation actions read the blocklist/allowlist and slur filters
/// multiple times. This causes a huge number of database reads if we hit the db directly. So we
/// cache these values for a short time, which will already make a huge difference and ensures that
/// changes take effect quickly.
const BLOCKLIST_CACHE_DURATION: Duration = Duration::from_secs(60);
static CONTEXT: Lazy<Vec<serde_json::Value>> = Lazy::new(|| { static CONTEXT: Lazy<Vec<serde_json::Value>> = Lazy::new(|| {
serde_json::from_str(include_str!("../assets/lemmy/context.json")).expect("parse context") serde_json::from_str(include_str!("../assets/lemmy/context.json")).expect("parse context")
@ -38,7 +46,7 @@ pub struct VerifyUrlData(pub DbPool);
#[async_trait] #[async_trait]
impl UrlVerifier for VerifyUrlData { impl UrlVerifier for VerifyUrlData {
async fn verify(&self, url: &Url) -> Result<(), &'static str> { async fn verify(&self, url: &Url) -> Result<(), &'static str> {
let local_site_data = fetch_local_site_data(&self.0) let local_site_data = local_site_data_cached(&self.0)
.await .await
.expect("read local site data"); .expect("read local site data");
check_apub_id_valid(url, &local_site_data)?; check_apub_id_valid(url, &local_site_data)?;
@ -53,9 +61,6 @@ impl UrlVerifier for VerifyUrlData {
/// - the correct scheme (either http or https) /// - the correct scheme (either http or https)
/// - URL being in the allowlist (if it is active) /// - URL being in the allowlist (if it is active)
/// - URL not being in the blocklist (if it is active) /// - URL not being in the blocklist (if it is active)
///
/// `use_strict_allowlist` should be true only when parsing a remote community, or when parsing a
/// post/comment in a local community.
#[tracing::instrument(skip(local_site_data))] #[tracing::instrument(skip(local_site_data))]
fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> Result<(), &'static str> { fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> Result<(), &'static str> {
let domain = apub_id.domain().expect("apud id has domain").to_string(); let domain = apub_id.domain().expect("apud id has domain").to_string();
@ -97,36 +102,50 @@ pub(crate) struct LocalSiteData {
blocked_instances: Vec<Instance>, blocked_instances: Vec<Instance>,
} }
pub(crate) async fn fetch_local_site_data( pub(crate) async fn local_site_data_cached(pool: &DbPool) -> LemmyResult<Arc<LocalSiteData>> {
pool: &DbPool, static CACHE: Lazy<Cache<(), Arc<LocalSiteData>>> = Lazy::new(|| {
) -> Result<LocalSiteData, diesel::result::Error> { Cache::builder()
// LocalSite may be missing .max_capacity(1)
let local_site = LocalSite::read(pool).await.ok(); .time_to_live(BLOCKLIST_CACHE_DURATION)
let allowed_instances = Instance::allowlist(pool).await?; .build()
let blocked_instances = Instance::blocklist(pool).await?; });
Ok(
CACHE
.try_get_with((), async {
let (local_site, allowed_instances, blocked_instances) = join3(
LocalSite::read(pool),
Instance::allowlist(pool),
Instance::blocklist(pool),
)
.await;
Ok(LocalSiteData { Ok::<_, diesel::result::Error>(Arc::new(LocalSiteData {
local_site, // LocalSite may be missing
allowed_instances, local_site: local_site.ok(),
blocked_instances, allowed_instances: allowed_instances?,
blocked_instances: blocked_instances?,
}))
}) })
.await?,
)
} }
#[tracing::instrument(skip(settings, local_site_data))] pub(crate) async fn check_apub_id_valid_with_strictness(
pub(crate) fn check_apub_id_valid_with_strictness(
apub_id: &Url, apub_id: &Url,
is_strict: bool, is_strict: bool,
local_site_data: &LocalSiteData, context: &LemmyContext,
settings: &Settings,
) -> Result<(), LemmyError> { ) -> Result<(), LemmyError> {
let domain = apub_id.domain().expect("apud id has domain").to_string(); let domain = apub_id.domain().expect("apud id has domain").to_string();
let local_instance = settings let local_instance = context
.settings()
.get_hostname_without_port() .get_hostname_without_port()
.expect("local hostname is valid"); .expect("local hostname is valid");
if domain == local_instance { if domain == local_instance {
return Ok(()); return Ok(());
} }
check_apub_id_valid(apub_id, local_site_data).map_err(LemmyError::from_message)?;
let local_site_data = local_site_data_cached(context.pool()).await?;
check_apub_id_valid(apub_id, &local_site_data).map_err(LemmyError::from_message)?;
// Only check allowlist if this is a community, and there are instances in the allowlist // Only check allowlist if this is a community, and there are instances in the allowlist
if is_strict && !local_site_data.allowed_instances.is_empty() { if is_strict && !local_site_data.allowed_instances.is_empty() {
@ -137,7 +156,8 @@ pub(crate) fn check_apub_id_valid_with_strictness(
.iter() .iter()
.map(|i| i.domain.clone()) .map(|i| i.domain.clone())
.collect::<Vec<String>>(); .collect::<Vec<String>>();
let local_instance = settings let local_instance = context
.settings()
.get_hostname_without_port() .get_hostname_without_port()
.expect("local hostname is valid"); .expect("local hostname is valid");
allowed_and_local.push(local_instance); allowed_and_local.push(local_instance);

View file

@ -1,7 +1,6 @@
use crate::{ use crate::{
activities::{verify_is_public, verify_person_in_community}, activities::{verify_is_public, verify_person_in_community},
check_apub_id_valid_with_strictness, check_apub_id_valid_with_strictness,
fetch_local_site_data,
mentions::collect_non_local_mentions, mentions::collect_non_local_mentions,
objects::{read_from_string_or_source, verify_is_remote_object}, objects::{read_from_string_or_source, verify_is_remote_object},
protocol::{ protocol::{
@ -132,14 +131,8 @@ impl Object for ApubComment {
verify_domains_match(note.attributed_to.inner(), note.id.inner())?; verify_domains_match(note.attributed_to.inner(), note.id.inner())?;
verify_is_public(&note.to, &note.cc)?; verify_is_public(&note.to, &note.cc)?;
let community = note.community(context).await?; let community = note.community(context).await?;
let local_site_data = fetch_local_site_data(context.pool()).await?;
check_apub_id_valid_with_strictness( check_apub_id_valid_with_strictness(note.id.inner(), community.local, context).await?;
note.id.inner(),
community.local,
&local_site_data,
context.settings(),
)?;
verify_is_remote_object(note.id.inner(), context.settings())?; verify_is_remote_object(note.id.inner(), context.settings())?;
verify_person_in_community(&note.attributed_to, &community, context).await?; verify_person_in_community(&note.attributed_to, &community, context).await?;
let (post, _) = note.get_parents(context).await?; let (post, _) = note.get_parents(context).await?;

View file

@ -1,6 +1,6 @@
use crate::{ use crate::{
check_apub_id_valid_with_strictness, check_apub_id_valid,
fetch_local_site_data, local_site_data_cached,
objects::instance::fetch_instance_actor_for_object, objects::instance::fetch_instance_actor_for_object,
protocol::{ protocol::{
objects::{group::Group, Endpoints, LanguageTag}, objects::{group::Group, Endpoints, LanguageTag},
@ -187,7 +187,7 @@ impl ApubCommunity {
) -> Result<Vec<Url>, LemmyError> { ) -> Result<Vec<Url>, LemmyError> {
let id = self.id; let id = self.id;
let local_site_data = fetch_local_site_data(context.pool()).await?; let local_site_data = local_site_data_cached(context.pool()).await?;
let follows = CommunityFollowerView::for_community(context.pool(), id).await?; let follows = CommunityFollowerView::for_community(context.pool(), id).await?;
let inboxes: Vec<Url> = follows let inboxes: Vec<Url> = follows
.into_iter() .into_iter()
@ -201,10 +201,7 @@ impl ApubCommunity {
.unique() .unique()
.filter(|inbox: &Url| inbox.host_str() != Some(&context.settings().hostname)) .filter(|inbox: &Url| inbox.host_str() != Some(&context.settings().hostname))
// Don't send to blocked instances // Don't send to blocked instances
.filter(|inbox| { .filter(|inbox| check_apub_id_valid(inbox, &local_site_data).is_ok())
check_apub_id_valid_with_strictness(inbox, false, &local_site_data, context.settings())
.is_ok()
})
.collect(); .collect();
Ok(inboxes) Ok(inboxes)

View file

@ -1,6 +1,6 @@
use crate::{ use crate::{
check_apub_id_valid_with_strictness, check_apub_id_valid_with_strictness,
fetch_local_site_data, local_site_data_cached,
objects::read_from_string_or_source_opt, objects::read_from_string_or_source_opt,
protocol::{ protocol::{
objects::{instance::Instance, LanguageTag}, objects::{instance::Instance, LanguageTag},
@ -113,15 +113,14 @@ impl Object for ApubSite {
expected_domain: &Url, expected_domain: &Url,
data: &Data<Self::DataType>, data: &Data<Self::DataType>,
) -> Result<(), LemmyError> { ) -> Result<(), LemmyError> {
let local_site_data = fetch_local_site_data(data.pool()).await?; check_apub_id_valid_with_strictness(apub.id.inner(), true, data).await?;
check_apub_id_valid_with_strictness(apub.id.inner(), true, &local_site_data, data.settings())?;
verify_domains_match(expected_domain, apub.id.inner())?; verify_domains_match(expected_domain, apub.id.inner())?;
let local_site_data = local_site_data_cached(data.pool()).await?;
let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site); let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site);
check_slurs(&apub.name, slur_regex)?; check_slurs(&apub.name, slur_regex)?;
check_slurs_opt(&apub.summary, slur_regex)?; check_slurs_opt(&apub.summary, slur_regex)?;
Ok(()) Ok(())
} }

View file

@ -1,6 +1,6 @@
use crate::{ use crate::{
check_apub_id_valid_with_strictness, check_apub_id_valid_with_strictness,
fetch_local_site_data, local_site_data_cached,
objects::{instance::fetch_instance_actor_for_object, read_from_string_or_source_opt}, objects::{instance::fetch_instance_actor_for_object, read_from_string_or_source_opt},
protocol::{ protocol::{
objects::{ objects::{
@ -118,19 +118,13 @@ impl Object for ApubPerson {
expected_domain: &Url, expected_domain: &Url,
context: &Data<Self::DataType>, context: &Data<Self::DataType>,
) -> Result<(), LemmyError> { ) -> Result<(), LemmyError> {
let local_site_data = fetch_local_site_data(context.pool()).await?; let local_site_data = local_site_data_cached(context.pool()).await?;
let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site); let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site);
check_slurs(&person.preferred_username, slur_regex)?; check_slurs(&person.preferred_username, slur_regex)?;
check_slurs_opt(&person.name, slur_regex)?; check_slurs_opt(&person.name, slur_regex)?;
verify_domains_match(person.id.inner(), expected_domain)?; verify_domains_match(person.id.inner(), expected_domain)?;
check_apub_id_valid_with_strictness( check_apub_id_valid_with_strictness(person.id.inner(), false, context).await?;
person.id.inner(),
false,
&local_site_data,
context.settings(),
)?;
let bio = read_from_string_or_source_opt(&person.summary, &None, &person.source); let bio = read_from_string_or_source_opt(&person.summary, &None, &person.source);
check_slurs_opt(&bio, slur_regex)?; check_slurs_opt(&bio, slur_regex)?;

View file

@ -1,7 +1,7 @@
use crate::{ use crate::{
activities::{verify_is_public, verify_person_in_community}, activities::{verify_is_public, verify_person_in_community},
check_apub_id_valid_with_strictness, check_apub_id_valid_with_strictness,
fetch_local_site_data, local_site_data_cached,
objects::{read_from_string_or_source_opt, verify_is_remote_object}, objects::{read_from_string_or_source_opt, verify_is_remote_object},
protocol::{ protocol::{
objects::{ objects::{
@ -143,17 +143,11 @@ impl Object for ApubPost {
verify_is_remote_object(page.id.inner(), context.settings())?; verify_is_remote_object(page.id.inner(), context.settings())?;
}; };
let local_site_data = fetch_local_site_data(context.pool()).await?;
let community = page.community(context).await?; let community = page.community(context).await?;
check_apub_id_valid_with_strictness( check_apub_id_valid_with_strictness(page.id.inner(), community.local, context).await?;
page.id.inner(),
community.local,
&local_site_data,
context.settings(),
)?;
verify_person_in_community(&page.creator()?, &community, context).await?; verify_person_in_community(&page.creator()?, &community, context).await?;
let local_site_data = local_site_data_cached(context.pool()).await?;
let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site); let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site);
check_slurs_opt(&page.name, slur_regex)?; check_slurs_opt(&page.name, slur_regex)?;

View file

@ -1,6 +1,5 @@
use crate::{ use crate::{
check_apub_id_valid_with_strictness, check_apub_id_valid_with_strictness,
fetch_local_site_data,
objects::read_from_string_or_source, objects::read_from_string_or_source,
protocol::{ protocol::{
objects::chat_message::{ChatMessage, ChatMessageType}, objects::chat_message::{ChatMessage, ChatMessageType},
@ -102,14 +101,7 @@ impl Object for ApubPrivateMessage {
verify_domains_match(note.id.inner(), expected_domain)?; verify_domains_match(note.id.inner(), expected_domain)?;
verify_domains_match(note.attributed_to.inner(), note.id.inner())?; verify_domains_match(note.attributed_to.inner(), note.id.inner())?;
let local_site_data = fetch_local_site_data(context.pool()).await?; check_apub_id_valid_with_strictness(note.id.inner(), false, context).await?;
check_apub_id_valid_with_strictness(
note.id.inner(),
false,
&local_site_data,
context.settings(),
)?;
let person = note.attributed_to.dereference(context).await?; let person = note.attributed_to.dereference(context).await?;
if person.banned { if person.banned {
return Err(LemmyError::from_message("Person is banned from site")); return Err(LemmyError::from_message("Person is banned from site"));

View file

@ -5,7 +5,7 @@ use crate::{
community_moderators::ApubCommunityModerators, community_moderators::ApubCommunityModerators,
community_outbox::ApubCommunityOutbox, community_outbox::ApubCommunityOutbox,
}, },
fetch_local_site_data, local_site_data_cached,
objects::{community::ApubCommunity, read_from_string_or_source_opt}, objects::{community::ApubCommunity, read_from_string_or_source_opt},
protocol::{ protocol::{
objects::{Endpoints, LanguageTag}, objects::{Endpoints, LanguageTag},
@ -80,16 +80,10 @@ impl Group {
expected_domain: &Url, expected_domain: &Url,
context: &LemmyContext, context: &LemmyContext,
) -> Result<(), LemmyError> { ) -> Result<(), LemmyError> {
let local_site_data = fetch_local_site_data(context.pool()).await?; check_apub_id_valid_with_strictness(self.id.inner(), true, context).await?;
check_apub_id_valid_with_strictness(
self.id.inner(),
true,
&local_site_data,
context.settings(),
)?;
verify_domains_match(expected_domain, self.id.inner())?; verify_domains_match(expected_domain, self.id.inner())?;
let local_site_data = local_site_data_cached(context.pool()).await?;
let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site); let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site);
check_slurs(&self.preferred_username, slur_regex)?; check_slurs(&self.preferred_username, slur_regex)?;