From 1b5437cbe3fe0b9726c5760c05f52054ab4e54c2 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 25 Apr 2023 19:28:06 -0400 Subject: [PATCH] Addressing slow profile queries. #2777 (#2830) * Addressing slow profile queries. #2777 * Addressing PR comments. --- crates/apub/src/fetcher/mod.rs | 4 +- crates/db_schema/src/impls/community.rs | 45 ++++++---- crates/db_schema/src/impls/person.rs | 87 ++++++++----------- crates/db_views/src/comment_report_view.rs | 7 +- crates/db_views/src/comment_view.rs | 25 ++---- crates/db_views/src/local_user_view.rs | 3 +- crates/db_views/src/post_report_view.rs | 15 +--- crates/db_views/src/post_view.rs | 39 ++++----- .../db_views_actor/src/comment_reply_view.rs | 21 ++--- .../src/community_moderator_view.rs | 2 - .../src/community_person_ban_view.rs | 7 +- .../db_views_actor/src/person_mention_view.rs | 7 +- docker/docker_update.sh | 2 +- .../down.sql | 21 +++++ .../up.sql | 31 +++++++ src/scheduled_tasks.rs | 37 +++++--- 16 files changed, 175 insertions(+), 178 deletions(-) create mode 100644 migrations/2023-04-23-164732_add_person_details_indexes/down.sql create mode 100644 migrations/2023-04-23-164732_add_person_details_indexes/up.sql diff --git a/crates/apub/src/fetcher/mod.rs b/crates/apub/src/fetcher/mod.rs index de244178e..4f72d1488 100644 --- a/crates/apub/src/fetcher/mod.rs +++ b/crates/apub/src/fetcher/mod.rs @@ -41,9 +41,7 @@ where .splitn(2, '@') .collect_tuple() .expect("invalid query"); - let name = name.to_string(); - let domain = format!("{}://{}", context.settings().get_protocol_string(), domain); - let actor = DbActor::read_from_name_and_domain(context.pool(), &name, &domain).await; + let actor = DbActor::read_from_name_and_domain(context.pool(), name, domain).await; if actor.is_ok() { Ok(actor?.into()) } else if local_user_view.is_some() { diff --git a/crates/db_schema/src/impls/community.rs b/crates/db_schema/src/impls/community.rs index e717b8a86..ad9e6d47d 100644 --- a/crates/db_schema/src/impls/community.rs +++ b/crates/db_schema/src/impls/community.rs @@ -1,6 +1,6 @@ use crate::{ newtypes::{CommunityId, DbUrl, PersonId}, - schema::community::dsl::{actor_id, community, deleted, local, name, removed}, + schema::{community, instance}, source::{ actor_language::CommunityLanguage, community::{ @@ -19,7 +19,7 @@ use crate::{ utils::{functions::lower, get_conn, DbPool}, SubscribedType, }; -use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl, TextExpressionMethods}; +use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl}; use diesel_async::RunQueryDsl; #[async_trait] @@ -29,12 +29,15 @@ impl Crud for Community { type IdType = CommunityId; async fn read(pool: &DbPool, community_id: CommunityId) -> Result { let conn = &mut get_conn(pool).await?; - community.find(community_id).first::(conn).await + community::table + .find(community_id) + .first::(conn) + .await } async fn delete(pool: &DbPool, community_id: CommunityId) -> Result { let conn = &mut get_conn(pool).await?; - diesel::delete(community.find(community_id)) + diesel::delete(community::table.find(community_id)) .execute(conn) .await } @@ -47,9 +50,9 @@ impl Crud for Community { }; // Can't do separate insert/update commands because InsertForm/UpdateForm aren't convertible - let community_ = insert_into(community) + let community_ = insert_into(community::table) .values(form) - .on_conflict(actor_id) + .on_conflict(community::actor_id) .do_update() .set(form) .get_result::(conn) @@ -69,7 +72,7 @@ impl Crud for Community { form: &Self::UpdateForm, ) -> Result { let conn = &mut get_conn(pool).await?; - diesel::update(community.find(community_id)) + diesel::update(community::table.find(community_id)) .set(form) .get_result::(conn) .await @@ -121,14 +124,14 @@ impl Community { use crate::schema::community::dsl::{featured_url, moderators_url}; use CollectionType::*; let conn = &mut get_conn(pool).await?; - let res = community + let res = community::table .filter(moderators_url.eq(url)) .first::(conn) .await; if let Ok(c) = res { return Ok((c, Moderators)); } - let res = community + let res = community::table .filter(featured_url.eq(url)) .first::(conn) .await; @@ -280,8 +283,8 @@ impl ApubActor for Community { async fn read_from_apub_id(pool: &DbPool, object_id: &DbUrl) -> Result, Error> { let conn = &mut get_conn(pool).await?; Ok( - community - .filter(actor_id.eq(object_id)) + community::table + .filter(community::actor_id.eq(object_id)) .first::(conn) .await .ok() @@ -295,12 +298,14 @@ impl ApubActor for Community { include_deleted: bool, ) -> Result { let conn = &mut get_conn(pool).await?; - let mut q = community + let mut q = community::table .into_boxed() - .filter(local.eq(true)) - .filter(lower(name).eq(lower(community_name))); + .filter(community::local.eq(true)) + .filter(lower(community::name).eq(community_name.to_lowercase())); if !include_deleted { - q = q.filter(deleted.eq(false)).filter(removed.eq(false)); + q = q + .filter(community::deleted.eq(false)) + .filter(community::removed.eq(false)); } q.first::(conn).await } @@ -308,12 +313,14 @@ impl ApubActor for Community { async fn read_from_name_and_domain( pool: &DbPool, community_name: &str, - protocol_domain: &str, + for_domain: &str, ) -> Result { let conn = &mut get_conn(pool).await?; - community - .filter(lower(name).eq(lower(community_name))) - .filter(actor_id.like(format!("{protocol_domain}%"))) + community::table + .inner_join(instance::table) + .filter(lower(community::name).eq(community_name.to_lowercase())) + .filter(instance::domain.eq(for_domain)) + .select(community::all_columns) .first::(conn) .await } diff --git a/crates/db_schema/src/impls/person.rs b/crates/db_schema/src/impls/person.rs index 2c315e9c8..d2a7b08ce 100644 --- a/crates/db_schema/src/impls/person.rs +++ b/crates/db_schema/src/impls/person.rs @@ -1,18 +1,6 @@ use crate::{ newtypes::{CommunityId, DbUrl, PersonId}, - schema::person::dsl::{ - actor_id, - avatar, - banner, - bio, - deleted, - display_name, - local, - matrix_user_id, - name, - person, - updated, - }, + schema::{instance, local_user, person, person_follower}, source::person::{ Person, PersonFollower, @@ -23,14 +11,7 @@ use crate::{ traits::{ApubActor, Crud, Followable}, utils::{functions::lower, get_conn, naive_now, DbPool}, }; -use diesel::{ - dsl::insert_into, - result::Error, - ExpressionMethods, - JoinOnDsl, - QueryDsl, - TextExpressionMethods, -}; +use diesel::{dsl::insert_into, result::Error, ExpressionMethods, JoinOnDsl, QueryDsl}; use diesel_async::RunQueryDsl; #[async_trait] @@ -40,21 +21,23 @@ impl Crud for Person { type IdType = PersonId; async fn read(pool: &DbPool, person_id: PersonId) -> Result { let conn = &mut get_conn(pool).await?; - person - .filter(deleted.eq(false)) + person::table + .filter(person::deleted.eq(false)) .find(person_id) .first::(conn) .await } async fn delete(pool: &DbPool, person_id: PersonId) -> Result { let conn = &mut get_conn(pool).await?; - diesel::delete(person.find(person_id)).execute(conn).await + diesel::delete(person::table.find(person_id)) + .execute(conn) + .await } async fn create(pool: &DbPool, form: &PersonInsertForm) -> Result { let conn = &mut get_conn(pool).await?; - insert_into(person) + insert_into(person::table) .values(form) - .on_conflict(actor_id) + .on_conflict(person::actor_id) .do_update() .set(form) .get_result::(conn) @@ -66,7 +49,7 @@ impl Crud for Person { form: &PersonUpdateForm, ) -> Result { let conn = &mut get_conn(pool).await?; - diesel::update(person.find(person_id)) + diesel::update(person::table.find(person_id)) .set(form) .get_result::(conn) .await @@ -75,7 +58,6 @@ impl Crud for Person { impl Person { pub async fn delete_account(pool: &DbPool, person_id: PersonId) -> Result { - use crate::schema::local_user; let conn = &mut get_conn(pool).await?; // Set the local user info to none @@ -87,15 +69,15 @@ impl Person { .execute(conn) .await?; - diesel::update(person.find(person_id)) + diesel::update(person::table.find(person_id)) .set(( - display_name.eq::>(None), - avatar.eq::>(None), - banner.eq::>(None), - bio.eq::>(None), - matrix_user_id.eq::>(None), - deleted.eq(true), - updated.eq(naive_now()), + person::display_name.eq::>(None), + person::avatar.eq::>(None), + person::banner.eq::>(None), + person::bio.eq::>(None), + person::matrix_user_id.eq::>(None), + person::deleted.eq(true), + person::updated.eq(naive_now()), )) .get_result::(conn) .await @@ -115,9 +97,9 @@ impl ApubActor for Person { async fn read_from_apub_id(pool: &DbPool, object_id: &DbUrl) -> Result, Error> { let conn = &mut get_conn(pool).await?; Ok( - person - .filter(deleted.eq(false)) - .filter(actor_id.eq(object_id)) + person::table + .filter(person::deleted.eq(false)) + .filter(person::actor_id.eq(object_id)) .first::(conn) .await .ok() @@ -131,12 +113,12 @@ impl ApubActor for Person { include_deleted: bool, ) -> Result { let conn = &mut get_conn(pool).await?; - let mut q = person + let mut q = person::table .into_boxed() - .filter(local.eq(true)) - .filter(lower(name).eq(lower(from_name))); + .filter(person::local.eq(true)) + .filter(lower(person::name).eq(from_name.to_lowercase())); if !include_deleted { - q = q.filter(deleted.eq(false)) + q = q.filter(person::deleted.eq(false)) } q.first::(conn).await } @@ -144,12 +126,15 @@ impl ApubActor for Person { async fn read_from_name_and_domain( pool: &DbPool, person_name: &str, - protocol_domain: &str, + for_domain: &str, ) -> Result { let conn = &mut get_conn(pool).await?; - person - .filter(lower(name).eq(lower(person_name))) - .filter(actor_id.like(format!("{protocol_domain}%"))) + + person::table + .inner_join(instance::table) + .filter(lower(person::name).eq(person_name.to_lowercase())) + .filter(instance::domain.eq(for_domain)) + .select(person::all_columns) .first::(conn) .await } @@ -186,12 +171,14 @@ impl Followable for PersonFollower { } impl PersonFollower { - pub async fn list_followers(pool: &DbPool, person_id_: PersonId) -> Result, Error> { - use crate::schema::{person, person_follower, person_follower::person_id}; + pub async fn list_followers( + pool: &DbPool, + for_person_id: PersonId, + ) -> Result, Error> { let conn = &mut get_conn(pool).await?; person_follower::table .inner_join(person::table.on(person_follower::follower_id.eq(person::id))) - .filter(person_id.eq(person_id_)) + .filter(person_follower::person_id.eq(for_person_id)) .select(person::all_columns) .load(conn) .await diff --git a/crates/db_views/src/comment_report_view.rs b/crates/db_views/src/comment_report_view.rs index df14ea559..a5b8d71d4 100644 --- a/crates/db_views/src/comment_report_view.rs +++ b/crates/db_views/src/comment_report_view.rs @@ -62,12 +62,7 @@ impl CommentReportView { community_person_ban::table.on( community::id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(comment::creator_id)), ), ) .left_join( diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index ea967fdff..0615d9517 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -1,6 +1,5 @@ use crate::structs::CommentView; use diesel::{ - dsl::now, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -88,12 +87,7 @@ impl CommentView { community_person_ban::table.on( community::id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(comment::creator_id)), ), ) .left_join( @@ -199,12 +193,7 @@ impl<'a> CommentQuery<'a> { community_person_ban::table.on( community::id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(comment::creator_id)), ), ) .left_join( @@ -279,6 +268,10 @@ impl<'a> CommentQuery<'a> { query = query.filter(comment::content.ilike(fuzzy_search(&search_term))); }; + if let Some(community_id) = self.community_id { + query = query.filter(post::community_id.eq(community_id)); + } + if let Some(listing_type) = self.listing_type { match listing_type { ListingType::Subscribed => { @@ -299,10 +292,6 @@ impl<'a> CommentQuery<'a> { ) } } - }; - - if let Some(community_id) = self.community_id { - query = query.filter(post::community_id.eq(community_id)); } if self.saved_only.unwrap_or(false) { @@ -610,6 +599,7 @@ mod tests { let read_comment_views_no_person = CommentQuery::builder() .pool(pool) + .sort(Some(CommentSortType::Hot)) .post_id(Some(data.inserted_post.id)) .build() .list() @@ -623,6 +613,7 @@ mod tests { let read_comment_views_with_person = CommentQuery::builder() .pool(pool) + .sort(Some(CommentSortType::Hot)) .post_id(Some(data.inserted_post.id)) .local_user(Some(&data.inserted_local_user)) .build() diff --git a/crates/db_views/src/local_user_view.rs b/crates/db_views/src/local_user_view.rs index 65d251605..f37986cc9 100644 --- a/crates/db_views/src/local_user_view.rs +++ b/crates/db_views/src/local_user_view.rs @@ -54,11 +54,10 @@ impl LocalUserView { }) } - // TODO check where this is used pub async fn read_from_name(pool: &DbPool, name: &str) -> Result { let conn = &mut get_conn(pool).await?; let (local_user, person, counts) = local_user::table - .filter(person::name.eq(name)) + .filter(lower(person::name).eq(name.to_lowercase())) .inner_join(person::table) .inner_join(person_aggregates::table.on(person::id.eq(person_aggregates::person_id))) .select(( diff --git a/crates/db_views/src/post_report_view.rs b/crates/db_views/src/post_report_view.rs index afda89e69..29136a351 100644 --- a/crates/db_views/src/post_report_view.rs +++ b/crates/db_views/src/post_report_view.rs @@ -1,6 +1,5 @@ use crate::structs::PostReportView; use diesel::{ - dsl::now, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -77,12 +76,7 @@ impl PostReportView { community_person_ban::table.on( post::community_id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(post::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(post::creator_id)), ), ) .left_join( @@ -194,12 +188,7 @@ impl<'a> PostReportQuery<'a> { community_person_ban::table.on( post::community_id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(post::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(post::creator_id)), ), ) .left_join( diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index 06cd4f294..f3319a90c 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -82,12 +82,7 @@ impl PostView { community_person_ban::table.on( post::community_id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(post::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(post::creator_id)), ), ) .inner_join(post_aggregates::table) @@ -230,12 +225,7 @@ impl<'a> PostQuery<'a> { community_person_ban::table.on( post::community_id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(post::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(post::creator_id)), ), ) .inner_join(post_aggregates::table) @@ -269,7 +259,7 @@ impl<'a> PostQuery<'a> { ) .left_join( community_block::table.on( - community::id + post::community_id .eq(community_block::community_id) .and(community_block::person_id.eq(person_id_join)), ), @@ -321,6 +311,18 @@ impl<'a> PostQuery<'a> { .filter(community::deleted.eq(false)); } + if self.community_id.is_none() { + query = query.then_order_by(post_aggregates::featured_local.desc()); + } else if let Some(community_id) = self.community_id { + query = query + .filter(post::community_id.eq(community_id)) + .then_order_by(post_aggregates::featured_community.desc()); + } + + if let Some(creator_id) = self.creator_id { + query = query.filter(post::creator_id.eq(creator_id)); + } + if let Some(listing_type) = self.listing_type { match listing_type { ListingType::Subscribed => { @@ -342,13 +344,6 @@ impl<'a> PostQuery<'a> { } } } - if self.community_id.is_none() { - query = query.then_order_by(post_aggregates::featured_local.desc()); - } else if let Some(community_id) = self.community_id { - query = query - .filter(post::community_id.eq(community_id)) - .then_order_by(post_aggregates::featured_community.desc()); - } if let Some(url_search) = self.url_search { query = query.filter(post::url.eq(url_search)); @@ -363,10 +358,6 @@ impl<'a> PostQuery<'a> { ); } - if let Some(creator_id) = self.creator_id { - query = query.filter(post::creator_id.eq(creator_id)); - } - if !self.local_user.map(|l| l.show_nsfw).unwrap_or(false) { query = query .filter(post::nsfw.eq(false)) diff --git a/crates/db_views_actor/src/comment_reply_view.rs b/crates/db_views_actor/src/comment_reply_view.rs index c9a3d9713..2a3c58d6c 100644 --- a/crates/db_views_actor/src/comment_reply_view.rs +++ b/crates/db_views_actor/src/comment_reply_view.rs @@ -1,6 +1,5 @@ use crate::structs::CommentReplyView; use diesel::{ - dsl::now, result::Error, BoolExpressionMethods, ExpressionMethods, @@ -91,12 +90,7 @@ impl CommentReplyView { community_person_ban::table.on( community::id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(comment::creator_id)), ), ) .left_join( @@ -212,12 +206,7 @@ impl<'a> CommentReplyQuery<'a> { community_person_ban::table.on( community::id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(comment::creator_id)), ), ) .left_join( @@ -276,12 +265,12 @@ impl<'a> CommentReplyQuery<'a> { query = query.filter(person::bot_account.eq(false)); }; - query = match self.sort.unwrap_or(CommentSortType::Hot) { + query = match self.sort.unwrap_or(CommentSortType::New) { CommentSortType::Hot => query .then_order_by(hot_rank(comment_aggregates::score, comment_aggregates::published).desc()) .then_order_by(comment_aggregates::published.desc()), - CommentSortType::New => query.then_order_by(comment::published.desc()), - CommentSortType::Old => query.then_order_by(comment::published.asc()), + CommentSortType::New => query.then_order_by(comment_reply::published.desc()), + CommentSortType::Old => query.then_order_by(comment_reply::published.asc()), CommentSortType::Top => query.order_by(comment_aggregates::score.desc()), }; diff --git a/crates/db_views_actor/src/community_moderator_view.rs b/crates/db_views_actor/src/community_moderator_view.rs index 70e799b85..afdfbfb28 100644 --- a/crates/db_views_actor/src/community_moderator_view.rs +++ b/crates/db_views_actor/src/community_moderator_view.rs @@ -19,7 +19,6 @@ impl CommunityModeratorView { .inner_join(person::table) .select((community::all_columns, person::all_columns)) .filter(community_moderator::community_id.eq(community_id)) - .order_by(community_moderator::published) .load::(conn) .await?; @@ -35,7 +34,6 @@ impl CommunityModeratorView { .filter(community_moderator::person_id.eq(person_id)) .filter(community::deleted.eq(false)) .filter(community::removed.eq(false)) - .order_by(community_moderator::published) .load::(conn) .await?; diff --git a/crates/db_views_actor/src/community_person_ban_view.rs b/crates/db_views_actor/src/community_person_ban_view.rs index ebfce9ba8..35eacf0f7 100644 --- a/crates/db_views_actor/src/community_person_ban_view.rs +++ b/crates/db_views_actor/src/community_person_ban_view.rs @@ -1,5 +1,5 @@ use crate::structs::CommunityPersonBanView; -use diesel::{dsl::now, result::Error, BoolExpressionMethods, ExpressionMethods, QueryDsl}; +use diesel::{result::Error, ExpressionMethods, QueryDsl}; use diesel_async::RunQueryDsl; use lemmy_db_schema::{ newtypes::{CommunityId, PersonId}, @@ -21,11 +21,6 @@ impl CommunityPersonBanView { .select((community::all_columns, person::all_columns)) .filter(community_person_ban::community_id.eq(from_community_id)) .filter(community_person_ban::person_id.eq(from_person_id)) - .filter( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ) .order_by(community_person_ban::published) .first::<(Community, Person)>(conn) .await?; diff --git a/crates/db_views_actor/src/person_mention_view.rs b/crates/db_views_actor/src/person_mention_view.rs index c0fefbe53..1b0e6415e 100644 --- a/crates/db_views_actor/src/person_mention_view.rs +++ b/crates/db_views_actor/src/person_mention_view.rs @@ -91,12 +91,7 @@ impl PersonMentionView { community_person_ban::table.on( community::id .eq(community_person_ban::community_id) - .and(community_person_ban::person_id.eq(comment::creator_id)) - .and( - community_person_ban::expires - .is_null() - .or(community_person_ban::expires.gt(now)), - ), + .and(community_person_ban::person_id.eq(comment::creator_id)), ), ) .left_join( diff --git a/docker/docker_update.sh b/docker/docker_update.sh index 64578fc9f..f2d7fa43a 100755 --- a/docker/docker_update.sh +++ b/docker/docker_update.sh @@ -3,4 +3,4 @@ set -e mkdir -p volumes/pictrs sudo chown -R 991:991 volumes/pictrs -sudo docker-compose up -d --build +sudo docker compose up -d --build diff --git a/migrations/2023-04-23-164732_add_person_details_indexes/down.sql b/migrations/2023-04-23-164732_add_person_details_indexes/down.sql new file mode 100644 index 000000000..67879de43 --- /dev/null +++ b/migrations/2023-04-23-164732_add_person_details_indexes/down.sql @@ -0,0 +1,21 @@ +drop index idx_person_lower_name; +drop index idx_community_lower_name; +drop index idx_community_moderator_published; +drop index idx_community_moderator_community; +drop index idx_community_moderator_person; +drop index idx_comment_saved_comment; +drop index idx_comment_saved_person; +drop index idx_community_block_community; +drop index idx_community_block_person; +drop index idx_community_follower_community; +drop index idx_community_follower_person; +drop index idx_person_block_person; +drop index idx_person_block_target; +drop index idx_post_language; +drop index idx_comment_language; +drop index idx_person_aggregates_person; +drop index idx_person_post_aggregates_post; +drop index idx_person_post_aggregates_person; +drop index idx_comment_reply_comment; +drop index idx_comment_reply_recipient; +drop index idx_comment_reply_published; \ No newline at end of file diff --git a/migrations/2023-04-23-164732_add_person_details_indexes/up.sql b/migrations/2023-04-23-164732_add_person_details_indexes/up.sql new file mode 100644 index 000000000..df3dde7e3 --- /dev/null +++ b/migrations/2023-04-23-164732_add_person_details_indexes/up.sql @@ -0,0 +1,31 @@ +-- Add a few indexes to speed up person details queries +create index idx_person_lower_name on person (lower(name)); +create index idx_community_lower_name on community (lower(name)); + +create index idx_community_moderator_published on community_moderator (published); +create index idx_community_moderator_community on community_moderator (community_id); +create index idx_community_moderator_person on community_moderator (person_id); + +create index idx_comment_saved_comment on comment_saved (comment_id); +create index idx_comment_saved_person on comment_saved (person_id); + +create index idx_community_block_community on community_block (community_id); +create index idx_community_block_person on community_block (person_id); + +create index idx_community_follower_community on community_follower (community_id); +create index idx_community_follower_person on community_follower (person_id); + +create index idx_person_block_person on person_block (person_id); +create index idx_person_block_target on person_block (target_id); + +create index idx_post_language on post (language_id); +create index idx_comment_language on comment (language_id); + +create index idx_person_aggregates_person on person_aggregates (person_id); + +create index idx_person_post_aggregates_post on person_post_aggregates (post_id); +create index idx_person_post_aggregates_person on person_post_aggregates (person_id); + +create index idx_comment_reply_comment on comment_reply (comment_id); +create index idx_comment_reply_recipient on comment_reply (recipient_id); +create index idx_comment_reply_published on comment_reply (published desc); diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index fd5c074d4..5d98baf9b 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -1,8 +1,14 @@ -use clokwerk::{Scheduler, TimeUnits}; +use clokwerk::{Scheduler, TimeUnits as CTimeUnits}; +use diesel::{ + dsl::{now, IntervalDsl}, + Connection, + ExpressionMethods, + QueryDsl, +}; // Import week days and WeekDay use diesel::{sql_query, PgConnection, RunQueryDsl}; -use diesel::{Connection, ExpressionMethods, QueryDsl}; use lemmy_db_schema::{ + schema::{activity, community_person_ban, instance, person}, source::instance::{Instance, InstanceForm}, utils::naive_now, }; @@ -27,7 +33,7 @@ pub fn setup(db_url: String, user_agent: String) -> Result<(), LemmyError> { // On startup, reindex the tables non-concurrently // TODO remove this for now, since it slows down startup a lot on lemmy.ml reindex_aggregates_tables(&mut conn, true); - scheduler.every(1.hour()).run(move || { + scheduler.every(CTimeUnits::hour(1)).run(move || { let conn = &mut PgConnection::establish(&db_url) .unwrap_or_else(|_| panic!("Error connecting to {db_url}")); active_counts(conn); @@ -37,12 +43,12 @@ pub fn setup(db_url: String, user_agent: String) -> Result<(), LemmyError> { }); clear_old_activities(&mut conn); - scheduler.every(1.weeks()).run(move || { + scheduler.every(CTimeUnits::weeks(1)).run(move || { clear_old_activities(&mut conn); }); update_instance_software(&mut conn_2, &user_agent); - scheduler.every(1.days()).run(move || { + scheduler.every(CTimeUnits::days(1)).run(move || { update_instance_software(&mut conn_2, &user_agent); }); @@ -76,10 +82,8 @@ fn reindex_table(conn: &mut PgConnection, table_name: &str, concurrently: bool) /// Clear old activities (this table gets very large) fn clear_old_activities(conn: &mut PgConnection) { - use diesel::dsl::{now, IntervalDsl}; - use lemmy_db_schema::schema::activity::dsl::{activity, published}; info!("Clearing old activities..."); - diesel::delete(activity.filter(published.lt(now - 6.months()))) + diesel::delete(activity::table.filter(activity::published.lt(now - 6.months()))) .execute(conn) .expect("clear old activities"); info!("Done."); @@ -117,11 +121,19 @@ fn active_counts(conn: &mut PgConnection) { /// Set banned to false after ban expires fn update_banned_when_expired(conn: &mut PgConnection) { info!("Updating banned column if it expires ..."); - let update_ban_expires_stmt = - "update person set banned = false where banned = true and ban_expires < now()"; - sql_query(update_ban_expires_stmt) + + diesel::update( + person::table + .filter(person::banned.eq(true)) + .filter(person::ban_expires.lt(now)), + ) + .set(person::banned.eq(false)) + .execute(conn) + .expect("update person.banned when expires"); + + diesel::delete(community_person_ban::table.filter(community_person_ban::expires.lt(now))) .execute(conn) - .expect("update banned when expires"); + .expect("remove community_ban expired rows"); } /// Drops the phantom CCNEW indexes created by postgres @@ -136,7 +148,6 @@ fn drop_ccnew_indexes(conn: &mut PgConnection) { /// Updates the instance software and version fn update_instance_software(conn: &mut PgConnection, user_agent: &str) { - use lemmy_db_schema::schema::instance; info!("Updating instances software and versions..."); let client = Client::builder()