From 78702b59fd56f767f3d5612bfd60e294979f91f8 Mon Sep 17 00:00:00 2001 From: dullbananas Date: Tue, 2 Jul 2024 08:23:21 -0700 Subject: [PATCH] Use trigger to generate apub URL in insert instead of update, and fix query planner options not being set when TLS is disabled (#4797) * Update create.rs * Update utils.rs * Update utils.sql * Update triggers.sql * Update utils.sql * Update create.rs * Update create.rs * Update create.rs * Update create.rs * Update create.rs * Update create.rs * Update create.rs * Update create.rs * Create up.sql * Update up.sql * Update triggers.sql * Update utils.rs * stuff * stuff * revert some changed files * Revert "revert some changed files" This reverts commit 028eabb4bdcf9eda65e0f315ca1c98f8765f9d7e. * revert the correct files * partial reverts * migration, tests, fix establish_connection * lint * pg_format --- crates/api_crud/src/comment/create.rs | 24 +---- crates/api_crud/src/post/create.rs | 28 +----- crates/api_crud/src/private_message/create.rs | 22 +---- .../db_schema/replaceable_schema/triggers.sql | 40 ++++++++ crates/db_schema/replaceable_schema/utils.sql | 7 ++ crates/db_schema/src/impls/comment.rs | 8 +- crates/db_schema/src/impls/post.rs | 5 +- crates/db_schema/src/impls/private_message.rs | 8 +- crates/db_schema/src/utils.rs | 91 +++++++++++-------- .../2024-06-24-000000_ap_id_triggers/down.sql | 9 ++ .../2024-06-24-000000_ap_id_triggers/up.sql | 9 ++ 11 files changed, 142 insertions(+), 109 deletions(-) create mode 100644 migrations/2024-06-24-000000_ap_id_triggers/down.sql create mode 100644 migrations/2024-06-24-000000_ap_id_triggers/up.sql diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index 636f83392..ff3efd946 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -8,20 +8,18 @@ use lemmy_api_common::{ utils::{ check_community_user_action, check_post_deleted_or_removed, - generate_local_apub_endpoint, get_url_blocklist, is_mod_or_admin, local_site_to_slur_regex, process_markdown, update_read_comments, - EndpointType, }, }; use lemmy_db_schema::{ impls::actor_language::default_post_language, source::{ actor_language::CommunityLanguage, - comment::{Comment, CommentInsertForm, CommentLike, CommentLikeForm, CommentUpdateForm}, + comment::{Comment, CommentInsertForm, CommentLike, CommentLikeForm}, comment_reply::{CommentReply, CommentReplyUpdateForm}, local_site::LocalSite, person_mention::{PersonMention, PersonMentionUpdateForm}, @@ -126,25 +124,7 @@ pub async fn create_comment( .await .with_lemmy_type(LemmyErrorType::CouldntCreateComment)?; - // Necessary to update the ap_id let inserted_comment_id = inserted_comment.id; - let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - - let apub_id = generate_local_apub_endpoint( - EndpointType::Comment, - &inserted_comment_id.to_string(), - &protocol_and_hostname, - )?; - let updated_comment = Comment::update( - &mut context.pool(), - inserted_comment_id, - &CommentUpdateForm { - ap_id: Some(apub_id), - ..Default::default() - }, - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntCreateComment)?; // Scan the comment for user mentions, add those rows let mentions = scrape_text_for_mentions(&content); @@ -170,7 +150,7 @@ pub async fn create_comment( .with_lemmy_type(LemmyErrorType::CouldntLikeComment)?; ActivityChannel::submit_activity( - SendActivityData::CreateComment(updated_comment.clone()), + SendActivityData::CreateComment(inserted_comment.clone()), &context, ) .await?; diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 0b0fad5dc..39ef8ce09 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -8,13 +8,11 @@ use lemmy_api_common::{ send_activity::SendActivityData, utils::{ check_community_user_action, - generate_local_apub_endpoint, get_url_blocklist, honeypot_check, local_site_to_slur_regex, mark_post_as_read, process_markdown_opt, - EndpointType, }, }; use lemmy_db_schema::{ @@ -23,7 +21,7 @@ use lemmy_db_schema::{ actor_language::CommunityLanguage, community::Community, local_site::LocalSite, - post::{Post, PostInsertForm, PostLike, PostLikeForm, PostUpdateForm}, + post::{Post, PostInsertForm, PostLike, PostLikeForm}, }, traits::{Crud, Likeable}, utils::diesel_url_create, @@ -147,26 +145,8 @@ pub async fn create_post( .await .with_lemmy_type(LemmyErrorType::CouldntCreatePost)?; - let inserted_post_id = inserted_post.id; - let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - let apub_id = generate_local_apub_endpoint( - EndpointType::Post, - &inserted_post_id.to_string(), - &protocol_and_hostname, - )?; - let updated_post = Post::update( - &mut context.pool(), - inserted_post_id, - &PostUpdateForm { - ap_id: Some(apub_id), - ..Default::default() - }, - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntCreatePost)?; - generate_post_link_metadata( - updated_post.clone(), + inserted_post.clone(), custom_thumbnail.map(Into::into), |post| Some(SendActivityData::CreatePost(post)), Some(local_site), @@ -189,11 +169,11 @@ pub async fn create_post( mark_post_as_read(person_id, post_id, &mut context.pool()).await?; - if let Some(url) = updated_post.url.clone() { + if let Some(url) = inserted_post.url.clone() { if community.visibility == CommunityVisibility::Public { spawn_try_task(async move { let mut webmention = - Webmention::new::(updated_post.ap_id.clone().into(), url.clone().into())?; + Webmention::new::(inserted_post.ap_id.clone().into(), url.clone().into())?; webmention.set_checked(true); match webmention .send() diff --git a/crates/api_crud/src/private_message/create.rs b/crates/api_crud/src/private_message/create.rs index 0381d196c..46908da6e 100644 --- a/crates/api_crud/src/private_message/create.rs +++ b/crates/api_crud/src/private_message/create.rs @@ -6,19 +6,17 @@ use lemmy_api_common::{ send_activity::{ActivityChannel, SendActivityData}, utils::{ check_person_block, - generate_local_apub_endpoint, get_interface_language, get_url_blocklist, local_site_to_slur_regex, process_markdown, send_email_to_user, - EndpointType, }, }; use lemmy_db_schema::{ source::{ local_site::LocalSite, - private_message::{PrivateMessage, PrivateMessageInsertForm, PrivateMessageUpdateForm}, + private_message::{PrivateMessage, PrivateMessageInsertForm}, }, traits::Crud, }; @@ -58,24 +56,6 @@ pub async fn create_private_message( .await .with_lemmy_type(LemmyErrorType::CouldntCreatePrivateMessage)?; - let inserted_private_message_id = inserted_private_message.id; - let protocol_and_hostname = context.settings().get_protocol_and_hostname(); - let apub_id = generate_local_apub_endpoint( - EndpointType::PrivateMessage, - &inserted_private_message_id.to_string(), - &protocol_and_hostname, - )?; - PrivateMessage::update( - &mut context.pool(), - inserted_private_message.id, - &PrivateMessageUpdateForm { - ap_id: Some(apub_id), - ..Default::default() - }, - ) - .await - .with_lemmy_type(LemmyErrorType::CouldntCreatePrivateMessage)?; - let view = PrivateMessageView::read(&mut context.pool(), inserted_private_message.id) .await? .ok_or(LemmyErrorType::CouldntFindPrivateMessage)?; diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index 87866e89c..973d3325f 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -564,6 +564,10 @@ BEGIN IF NOT (NEW.path ~ ('*.' || id)::lquery) THEN NEW.path = NEW.path || id; END IF; + -- Set local ap_id + IF NEW.local THEN + NEW.ap_id = coalesce(NEW.ap_id, r.local_url ('/comment/' || id)); + END IF; RETURN NEW; END $$; @@ -573,3 +577,39 @@ CREATE TRIGGER change_values FOR EACH ROW EXECUTE FUNCTION r.comment_change_values (); +CREATE FUNCTION r.post_change_values () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + -- Set local ap_id + IF NEW.local THEN + NEW.ap_id = coalesce(NEW.ap_id, r.local_url ('/post/' || NEW.id::text)); + END IF; + RETURN NEW; +END +$$; + +CREATE TRIGGER change_values + BEFORE INSERT ON post + FOR EACH ROW + EXECUTE FUNCTION r.post_change_values (); + +CREATE FUNCTION r.private_message_change_values () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + -- Set local ap_id + IF NEW.local THEN + NEW.ap_id = coalesce(NEW.ap_id, r.local_url ('/private_message/' || NEW.id::text)); + END IF; + RETURN NEW; +END +$$; + +CREATE TRIGGER change_values + BEFORE INSERT ON private_message + FOR EACH ROW + EXECUTE FUNCTION r.private_message_change_values (); + diff --git a/crates/db_schema/replaceable_schema/utils.sql b/crates/db_schema/replaceable_schema/utils.sql index 26447f2c2..c766d25f2 100644 --- a/crates/db_schema/replaceable_schema/utils.sql +++ b/crates/db_schema/replaceable_schema/utils.sql @@ -57,6 +57,13 @@ BEGIN END; $$; +CREATE FUNCTION r.local_url (url_path text) + RETURNS text + LANGUAGE sql + STABLE PARALLEL SAFE RETURN ( +current_setting('lemmy.protocol_and_hostname') || url_path +); + -- This function creates statement-level triggers for all operation types. It's designed this way -- because of these limitations: -- * A trigger that uses transition tables can only handle 1 operation type. diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 0ffd53f86..aa7b418fe 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -223,6 +223,7 @@ mod tests { use diesel_ltree::Ltree; use pretty_assertions::assert_eq; use serial_test::serial; + use url::Url; #[tokio::test] #[serial] @@ -273,7 +274,12 @@ mod tests { path: Ltree(format!("0.{}", inserted_comment.id)), published: inserted_comment.published, updated: None, - ap_id: inserted_comment.ap_id.clone(), + ap_id: Url::parse(&format!( + "https://lemmy-alpha/comment/{}", + inserted_comment.id + )) + .unwrap() + .into(), distinguished: false, local: true, language_id: LanguageId::default(), diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index ac6cf76aa..8e14bee9f 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -390,6 +390,7 @@ mod tests { use pretty_assertions::assert_eq; use serial_test::serial; use std::collections::HashSet; + use url::Url; #[tokio::test] #[serial] @@ -447,7 +448,9 @@ mod tests { embed_description: None, embed_video_url: None, thumbnail_url: None, - ap_id: inserted_post.ap_id.clone(), + ap_id: Url::parse(&format!("https://lemmy-alpha/post/{}", inserted_post.id)) + .unwrap() + .into(), local: true, language_id: Default::default(), featured_community: false, diff --git a/crates/db_schema/src/impls/private_message.rs b/crates/db_schema/src/impls/private_message.rs index 3cbfd052d..fe3629a1a 100644 --- a/crates/db_schema/src/impls/private_message.rs +++ b/crates/db_schema/src/impls/private_message.rs @@ -100,6 +100,7 @@ mod tests { }; use pretty_assertions::assert_eq; use serial_test::serial; + use url::Url; #[tokio::test] #[serial] @@ -138,7 +139,12 @@ mod tests { read: false, updated: None, published: inserted_private_message.published, - ap_id: inserted_private_message.ap_id.clone(), + ap_id: Url::parse(&format!( + "https://lemmy-alpha/private_message/{}", + inserted_private_message.id + )) + .unwrap() + .into(), local: true, }; diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 6761adeee..cca449b0e 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -30,7 +30,8 @@ use diesel_async::{ AsyncDieselConnectionManager, ManagerConfig, }, - SimpleAsyncConnection, + AsyncConnection, + RunQueryDsl, }; use futures_util::{future::BoxFuture, Future, FutureExt}; use i_love_jesus::CursorKey; @@ -332,34 +333,50 @@ pub fn diesel_url_create(opt: Option<&str>) -> LemmyResult> { fn establish_connection(config: &str) -> BoxFuture> { let fut = async { - rustls::crypto::ring::default_provider() - .install_default() - .expect("Failed to install rustls crypto provider"); + // We only support TLS with sslmode=require currently + let mut conn = if config.contains("sslmode=require") { + rustls::crypto::ring::default_provider() + .install_default() + .expect("Failed to install rustls crypto provider"); - let rustls_config = DangerousClientConfigBuilder { - cfg: ClientConfig::builder(), - } - .with_custom_certificate_verifier(Arc::new(NoCertVerifier {})) - .with_no_client_auth(); - - let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config); - let (client, conn) = tokio_postgres::connect(config, tls) - .await - .map_err(|e| ConnectionError::BadConnection(e.to_string()))?; - tokio::spawn(async move { - if let Err(e) = conn.await { - error!("Database connection failed: {e}"); + let rustls_config = DangerousClientConfigBuilder { + cfg: ClientConfig::builder(), } - }); - let mut conn = AsyncPgConnection::try_from(client).await?; - // * Change geqo_threshold back to default value if it was changed, so it's higher than the - // collapse limits - // * Change collapse limits from 8 to 11 so the query planner can find a better table join order - // for more complicated queries - conn - .batch_execute("SET geqo_threshold=12;SET from_collapse_limit=11;SET join_collapse_limit=11;") - .await - .map_err(ConnectionError::CouldntSetupConfiguration)?; + .with_custom_certificate_verifier(Arc::new(NoCertVerifier {})) + .with_no_client_auth(); + + let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config); + let (client, conn) = tokio_postgres::connect(config, tls) + .await + .map_err(|e| ConnectionError::BadConnection(e.to_string()))?; + tokio::spawn(async move { + if let Err(e) = conn.await { + error!("Database connection failed: {e}"); + } + }); + AsyncPgConnection::try_from(client).await? + } else { + AsyncPgConnection::establish(config).await? + }; + + diesel::select(( + // Change geqo_threshold back to default value if it was changed, so it's higher than the + // collapse limits + functions::set_config("geqo_threshold", "12", false), + // Change collapse limits from 8 to 11 so the query planner can find a better table join + // order for more complicated queries + functions::set_config("from_collapse_limit", "11", false), + functions::set_config("join_collapse_limit", "11", false), + // Set `lemmy.protocol_and_hostname` so triggers can use it + functions::set_config( + "lemmy.protocol_and_hostname", + SETTINGS.get_protocol_and_hostname(), + false, + ), + )) + .execute(&mut conn) + .await + .map_err(ConnectionError::CouldntSetupConfiguration)?; Ok(conn) }; fut.boxed() @@ -418,17 +435,11 @@ impl ServerCertVerifier for NoCertVerifier { pub async fn build_db_pool() -> LemmyResult { let db_url = SETTINGS.get_database_url(); - // We only support TLS with sslmode=require currently - let tls_enabled = db_url.contains("sslmode=require"); - let manager = if tls_enabled { - // diesel-async does not support any TLS connections out of the box, so we need to manually - // provide a setup function which handles creating the connection - let mut config = ManagerConfig::default(); - config.custom_setup = Box::new(establish_connection); - AsyncDieselConnectionManager::::new_with_config(&db_url, config) - } else { - AsyncDieselConnectionManager::::new(&db_url) - }; + // diesel-async does not support any TLS connections out of the box, so we need to manually + // provide a setup function which handles creating the connection + let mut config = ManagerConfig::default(); + config.custom_setup = Box::new(establish_connection); + let manager = AsyncDieselConnectionManager::::new_with_config(&db_url, config); let pool = Pool::builder(manager) .max_size(SETTINGS.database.pool_size) .runtime(Runtime::Tokio1) @@ -485,7 +496,7 @@ static EMAIL_REGEX: Lazy = Lazy::new(|| { }); pub mod functions { - use diesel::sql_types::{BigInt, Text, Timestamptz}; + use diesel::sql_types::{BigInt, Bool, Text, Timestamptz}; sql_function! { #[sql_name = "r.hot_rank"] @@ -508,6 +519,8 @@ pub mod functions { // really this function is variadic, this just adds the two-argument version sql_function!(fn coalesce(x: diesel::sql_types::Nullable, y: T) -> T); + + sql_function!(fn set_config(setting_name: Text, new_value: Text, is_local: Bool) -> Text); } pub const DELETED_REPLACEMENT_TEXT: &str = "*Permanently Deleted*"; diff --git a/migrations/2024-06-24-000000_ap_id_triggers/down.sql b/migrations/2024-06-24-000000_ap_id_triggers/down.sql new file mode 100644 index 000000000..72312eccf --- /dev/null +++ b/migrations/2024-06-24-000000_ap_id_triggers/down.sql @@ -0,0 +1,9 @@ +ALTER TABLE comment + ALTER COLUMN ap_id SET DEFAULT generate_unique_changeme (); + +ALTER TABLE post + ALTER COLUMN ap_id SET DEFAULT generate_unique_changeme (); + +ALTER TABLE private_message + ALTER COLUMN ap_id SET DEFAULT generate_unique_changeme (); + diff --git a/migrations/2024-06-24-000000_ap_id_triggers/up.sql b/migrations/2024-06-24-000000_ap_id_triggers/up.sql new file mode 100644 index 000000000..86e266d46 --- /dev/null +++ b/migrations/2024-06-24-000000_ap_id_triggers/up.sql @@ -0,0 +1,9 @@ +ALTER TABLE comment + ALTER COLUMN ap_id DROP DEFAULT; + +ALTER TABLE post + ALTER COLUMN ap_id DROP DEFAULT; + +ALTER TABLE private_message + ALTER COLUMN ap_id DROP DEFAULT; +