Avoid race condition when streaming deleted statuses (#10280)

* Avoid race condition when streaming deleted statuses

* Move redis lock to DistributionWorker to avoid extra Redis value

app/services/remove_status_service.rb

@@ -14,6 +14,8 @@ class RemoveStatusService < BaseService
     @stream_entry = status.stream_entry
     @options      = options
 
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
         remove_from_self if status.account.local?
         remove_from_followers
         remove_from_lists
@@ -24,6 +26,10 @@ class RemoveStatusService < BaseService
         remove_from_media if status.media_attachments.any?
 
         @status.destroy!
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
 
     # There is no reason to send out Undo activities when the
     # cause is that the original object has been removed, since
@@ -156,4 +162,8 @@ class RemoveStatusService < BaseService
     redis.publish('timeline:public:media', @payload)
     redis.publish('timeline:public:local:media', @payload) if @status.local?
   end
+
+  def lock_options
+    { redis: Redis.current, key: "distribute:#{@status.id}" }
+  end
 end

app/workers/distribution_worker.rb

@@ -4,7 +4,13 @@ class DistributionWorker
   include Sidekiq::Worker
 
   def perform(status_id)
+    RedisLock.acquire(redis: Redis.current, key: "distribute:#{status_id}") do |lock|
+      if lock.acquired?
         FanOutOnWriteService.new.call(Status.find(status_id))
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
   rescue ActiveRecord::RecordNotFound
     true
   end