change sidekiq queueing to bulk push (#3536)

Gemfile

@@ -52,6 +52,7 @@ gem 'sanitize', '~> 4.4'
 gem 'sidekiq', '~> 5.0'
 gem 'sidekiq-scheduler', '~> 2.1'
 gem 'sidekiq-unique-jobs', '~> 5.0'
+gem 'sidekiq-bulk', '~>0.1.1'
 gem 'simple-navigation', '~> 4.0'
 gem 'simple_form', '~> 3.4'
 gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie'

Gemfile.lock

@@ -402,6 +402,9 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.3, >= 3.3.3)
+    sidekiq-bulk (0.1.1)
+      activesupport
+      sidekiq
     sidekiq-scheduler (2.1.4)
       redis (~> 3)
       rufus-scheduler (~> 3.2)
@@ -543,6 +546,7 @@ DEPENDENCIES
   sanitize (~> 4.4)
   scss_lint (~> 0.53)
   sidekiq (~> 5.0)
+  sidekiq-bulk (~> 0.1.1)
   sidekiq-scheduler (~> 2.1)
   sidekiq-unique-jobs (~> 5.0)
   simple-navigation (~> 4.0)

app/controllers/settings/follower_domains_controller.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'sidekiq-bulk'
+
 class Settings::FollowerDomainsController < ApplicationController
   layout 'admin'
 
@@ -13,8 +15,8 @@ class Settings::FollowerDomainsController < ApplicationController
   def update
     domains = bulk_params[:select] || []
 
-    domains.each do |domain|
+    SoftBlockDomainFollowersWorker.push_bulk(domains) do |domain|
-      SoftBlockDomainFollowersWorker.perform_async(current_account.id, domain)
+      [current_account.id, domain]
     end
 
     redirect_to settings_follower_domains_path, notice: I18n.t('followers.success', count: domains.size)

app/services/fan_out_on_write_service.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'sidekiq-bulk'
+
 class FanOutOnWriteService < BaseService
   # Push a status into home and mentions feeds
   # @param [Status] status
@@ -34,8 +36,10 @@ class FanOutOnWriteService < BaseService
   def deliver_to_followers(status)
     Rails.logger.debug "Delivering status #{status.id} to followers"
 
-    status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_each do |follower|
+    status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers|
-      FeedInsertWorker.perform_async(status.id, follower.id)
+      FeedInsertWorker.push_bulk(followers) do |follower|
+        [status.id, follower.id]
+      end
     end
   end
 

app/workers/scheduler/subscriptions_scheduler.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
+
 require 'sidekiq-scheduler'
+require 'sidekiq-bulk'
 
 class Scheduler::SubscriptionsScheduler
   include Sidekiq::Worker
@@ -7,9 +9,7 @@ class Scheduler::SubscriptionsScheduler
   def perform
     logger.info 'Queueing PuSH re-subscriptions'
 
-    expiring_accounts.pluck(:id).each do |id|
+    Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id))
-      Pubsubhubbub::SubscribeWorker.perform_async(id)
-    end
   end
 
   private

app/workers/soft_block_domain_followers_worker.rb

@@ -1,13 +1,16 @@
 # frozen_string_literal: true
 
+require 'sidekiq-bulk'
+
 class SoftBlockDomainFollowersWorker
   include Sidekiq::Worker
 
   sidekiq_options queue: 'pull'
 
   def perform(account_id, domain)
-    Account.find(account_id).followers.where(domain: domain).pluck(:id).each do |follower_id|
+    followers_id = Account.find(account_id).followers.where(domain: domain).pluck(:id)
-      SoftBlockWorker.perform_async(account_id, follower_id)
+    SoftBlockWorker.push_bulk(followers_id) do |follower_id|
+      [account_id, follower_id]
     end
   end
 end