Throttle account deletions in tootctl self-destruct to avoid overwhelming Sidekiq/Redis

This commit is contained in:
Claire 2023-08-10 13:13:59 +02:00
parent ef2a6bc646
commit d7a42ae716

View file

@ -20,6 +20,8 @@ require_relative 'upgrade'
module Mastodon::CLI module Mastodon::CLI
class Main < Base class Main < Base
include Redisable
desc 'media SUBCOMMAND ...ARGS', 'Manage media files' desc 'media SUBCOMMAND ...ARGS', 'Manage media files'
subcommand 'media', Media subcommand 'media', Media
@ -137,8 +139,21 @@ module Mastodon::CLI
processed += 1 processed += 1
end end
Account.local.without_suspended.find_each { |account| delete_account.call(account) } Account.local.without_suspended.find_in_batches(batch_size: 50) do |accounts|
Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) } accounts.each { |account| delete_account.call(account) }
prompt.ok("Processed accounts so far: #{processed}")
sleep 5 while sidekiq_overwhelmed?
end
Account.local.suspended.joins(:deletion_request).find_in_batches(batch_size: 50) do |accounts|
accounts.each { |account| delete_account.call(account) }
prompt.ok("Processed accounts so far: #{processed}")
sleep 5 while sidekiq_overwhelmed?
end
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}") prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}")
prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data') prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
@ -152,5 +167,13 @@ module Mastodon::CLI
def version def version
say(Mastodon::Version.to_s) say(Mastodon::Version.to_s)
end end
private
def sidekiq_overwhelmed?
redis_mem_info = Sidekiq.redis_info
Sidekiq::Stats.new.enqueued > 5000 || redis_mem_info['used_memory'].to_f * 2 > redis_mem_info['total_system_memory'].to_f
end
end end
end end