From 1d0ad558ff51ce3e42ef4e4c9a9bed3c4b722d0d Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 15 Mar 2023 03:45:15 +0100 Subject: [PATCH] Change sidekiq-bulk's batch size from 10,000 to 1,000 jobs in one Redis call (#24034) --- app/lib/activitypub/forwarder.rb | 2 +- app/services/delete_account_service.rb | 4 ++-- app/services/remove_status_service.rb | 2 +- app/services/suspend_account_service.rb | 12 ++++++------ app/services/unsuspend_account_service.rb | 2 +- app/services/update_account_service.rb | 2 +- .../activitypub/distribute_poll_update_worker.rb | 2 +- app/workers/activitypub/move_distribution_worker.rb | 2 +- app/workers/activitypub/raw_distribution_worker.rb | 2 +- lib/cli.rb | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/lib/activitypub/forwarder.rb b/app/lib/activitypub/forwarder.rb index b01d63e58..3a94f9669 100644 --- a/app/lib/activitypub/forwarder.rb +++ b/app/lib/activitypub/forwarder.rb @@ -12,7 +12,7 @@ class ActivityPub::Forwarder end def forward! - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, signature_account_id, inbox_url] end end diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index a2d535d26..190a72e5c 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -257,11 +257,11 @@ class DeleteAccountService < BaseService end def delete_actor! - ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes, limit: 1_000) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end - ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes) do |inbox_url| + ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes, limit: 1_000) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 45cfb75f4..ea799db57 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -88,7 +88,7 @@ class RemoveStatusService < BaseService status_reach_finder = StatusReachFinder.new(@status, unsafe: true) - ActivityPub::DeliveryWorker.push_bulk(status_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(status_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 211544fea..cfb3eb583 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -31,13 +31,13 @@ class SuspendAccountService < BaseService # counterpart to this operation, i.e. you can't then force a remote # account to re-follow you, so this part is not reversible. - follows = Follow.where(account: @account).to_a + Follow.where(account: @account).find_in_batches do |follows| + ActivityPub::DeliveryWorker.push_bulk(follows) do |follow| + [Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), follow.target_account_id, @account.inbox_url] + end - ActivityPub::DeliveryWorker.push_bulk(follows) do |follow| - [Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), follow.target_account_id, @account.inbox_url] + follows.each(&:destroy) end - - follows.each(&:destroy) end def distribute_update_actor! @@ -45,7 +45,7 @@ class SuspendAccountService < BaseService account_reach_finder = AccountReachFinder.new(@account) - ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb index 70667308e..d851a0f70 100644 --- a/app/services/unsuspend_account_service.rb +++ b/app/services/unsuspend_account_service.rb @@ -41,7 +41,7 @@ class UnsuspendAccountService < BaseService account_reach_finder = AccountReachFinder.new(@account) - ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes, limit: 1_000) do |inbox_url| [signed_activity_json, @account.id, inbox_url] end end diff --git a/app/services/update_account_service.rb b/app/services/update_account_service.rb index 71976ab00..4604d71b2 100644 --- a/app/services/update_account_service.rb +++ b/app/services/update_account_service.rb @@ -22,7 +22,7 @@ class UpdateAccountService < BaseService def authorize_all_follow_requests(account) follow_requests = FollowRequest.where(target_account: account) follow_requests = follow_requests.preload(:account).select { |req| !req.account.silenced? } - AuthorizeFollowWorker.push_bulk(follow_requests) do |req| + AuthorizeFollowWorker.push_bulk(follow_requests, limit: 1_000) do |req| [req.account_id, req.target_account_id] end end diff --git a/app/workers/activitypub/distribute_poll_update_worker.rb b/app/workers/activitypub/distribute_poll_update_worker.rb index 25dee4ee2..8c1eefd93 100644 --- a/app/workers/activitypub/distribute_poll_update_worker.rb +++ b/app/workers/activitypub/distribute_poll_update_worker.rb @@ -12,7 +12,7 @@ class ActivityPub::DistributePollUpdateWorker return unless @status.preloadable_poll - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, @account.id, inbox_url] end diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb index 65c5c0d1c..1680fcc76 100644 --- a/app/workers/activitypub/move_distribution_worker.rb +++ b/app/workers/activitypub/move_distribution_worker.rb @@ -10,7 +10,7 @@ class ActivityPub::MoveDistributionWorker @migration = AccountMigration.find(migration_id) @account = @migration.account - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [signed_payload, @account.id, inbox_url] end diff --git a/app/workers/activitypub/raw_distribution_worker.rb b/app/workers/activitypub/raw_distribution_worker.rb index 8ecc17db9..c77821e0f 100644 --- a/app/workers/activitypub/raw_distribution_worker.rb +++ b/app/workers/activitypub/raw_distribution_worker.rb @@ -25,7 +25,7 @@ class ActivityPub::RawDistributionWorker def distribute! return if inboxes.empty? - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [payload, source_account_id, inbox_url, options] end end diff --git a/lib/cli.rb b/lib/cli.rb index 157465c4b..ac235cf03 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -131,7 +131,7 @@ module Mastodon json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account)) unless options[:dry_run] - ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url| [json, account.id, inbox_url] end