specter/mastodon
1
0
Fork 0
forked from fedi/mastodon
Code Pull requests Activity

Change indexing frequency from 5 minutes to 1 minute, add locks to schedulers (#26304)

Browse source
This commit is contained in:
Eugen Rochko 2023-08-03 11:04:05 +02:00 committed by GitHub
parent f55f0ab0c3
commit a0fad5c8bb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 11 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/workers/scheduler/follow_recommendations_scheduler.rb
View file

@ -4,7 +4,7 @@ class Scheduler::FollowRecommendationsScheduler
include Sidekiq::Worker include Sidekiq::Worker
include Redisable include Redisable
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
# The maximum number of accounts that can be requested in one page from the # The maximum number of accounts that can be requested in one page from the
# API is 80, and the suggestions API does not allow pagination. This number # API is 80, and the suggestions API does not allow pagination. This number

6
app/workers/scheduler/indexing_scheduler.rb
View file

@ -4,7 +4,7 @@ class Scheduler::IndexingScheduler
include Sidekiq::Worker include Sidekiq::Worker
include Redisable include Redisable
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
IMPORT_BATCH_SIZE = 1000 IMPORT_BATCH_SIZE = 1000
SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE
@ -16,9 +16,7 @@ class Scheduler::IndexingScheduler
with_redis do |redis| with_redis do |redis|
redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
type.import!(ids) type.import!(ids)
redis.pipelined do |pipeline| redis.srem("chewy:queue:#{type.name}", ids)
pipeline.srem("chewy:queue:#{type.name}", ids)
end
end end
end end
end end

2
app/workers/scheduler/instance_refresh_scheduler.rb
View file

@ -3,7 +3,7 @@
class Scheduler::InstanceRefreshScheduler class Scheduler::InstanceRefreshScheduler
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform def perform
Instance.refresh Instance.refresh

2
app/workers/scheduler/ip_cleanup_scheduler.rb
View file

@ -6,7 +6,7 @@ class Scheduler::IpCleanupScheduler
IP_RETENTION_PERIOD = ENV.fetch('IP_RETENTION_PERIOD', 1.year).to_i.seconds.freeze IP_RETENTION_PERIOD = ENV.fetch('IP_RETENTION_PERIOD', 1.year).to_i.seconds.freeze
SESSION_RETENTION_PERIOD = ENV.fetch('SESSION_RETENTION_PERIOD', 1.year).to_i.seconds.freeze SESSION_RETENTION_PERIOD = ENV.fetch('SESSION_RETENTION_PERIOD', 1.year).to_i.seconds.freeze
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform def perform
clean_ip_columns! clean_ip_columns!

2
app/workers/scheduler/pghero_scheduler.rb
View file

@ -3,7 +3,7 @@
class Scheduler::PgheroScheduler class Scheduler::PgheroScheduler
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform def perform
PgHero.capture_space_stats PgHero.capture_space_stats

2
app/workers/scheduler/scheduled_statuses_scheduler.rb
View file

@ -3,7 +3,7 @@
class Scheduler::ScheduledStatusesScheduler class Scheduler::ScheduledStatusesScheduler
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform def perform
publish_scheduled_statuses! publish_scheduled_statuses!

2
app/workers/scheduler/suspended_user_cleanup_scheduler.rb
View file

@ -16,7 +16,7 @@ class Scheduler::SuspendedUserCleanupScheduler
# has the capacity for it. # has the capacity for it.
MAX_DELETIONS_PER_JOB = 10 MAX_DELETIONS_PER_JOB = 10
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform def perform
return if Sidekiq::Queue.new('pull').size > MAX_PULL_SIZE return if Sidekiq::Queue.new('pull').size > MAX_PULL_SIZE

2
app/workers/scheduler/user_cleanup_scheduler.rb
View file

@ -3,7 +3,7 @@
class Scheduler::UserCleanupScheduler class Scheduler::UserCleanupScheduler
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options retry: 0 sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform def perform
clean_unconfirmed_accounts! clean_unconfirmed_accounts!

2
app/workers/scheduler/vacuum_scheduler.rb
View file

@ -3,7 +3,7 @@
class Scheduler::VacuumScheduler class Scheduler::VacuumScheduler
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options retry: 0, lock: :until_executed sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform def perform
vacuum_operations.each do |operation| vacuum_operations.each do |operation|

2
config/sidekiq.yml
View file

@ -23,7 +23,7 @@
class: Scheduler::Trends::ReviewNotificationsScheduler class: Scheduler::Trends::ReviewNotificationsScheduler
queue: scheduler queue: scheduler
indexing_scheduler: indexing_scheduler:
every: '5m' interval: 1 minute
class: Scheduler::IndexingScheduler class: Scheduler::IndexingScheduler
queue: scheduler queue: scheduler
vacuum_scheduler: vacuum_scheduler: