mastodon/app/lib/attachment_batch.rb

# frozen_string_literal: true

class AttachmentBatch
  # Maximum amount of objects you can delete in an S3 API call. It's
  # important to remember that this does not correspond to the number
  # of records in the batch, since records can have multiple attachments
  LIMIT = ENV.fetch('S3_BATCH_DELETE_LIMIT', 1000).to_i
  MAX_RETRY = ENV.fetch('S3_BATCH_DELETE_RETRY', 3).to_i

  # Attributes generated and maintained by Paperclip (not all of them
  # are always used on every class, however)
  NULLABLE_ATTRIBUTES = %w(
    file_name
    content_type
    file_size
    fingerprint
    created_at
    updated_at
  ).freeze

  # Styles that are always present even when not explicitly defined
  BASE_STYLES = %i(original).freeze

  attr_reader :klass, :records, :storage_mode

  def initialize(klass, records)
    @klass            = klass
    @records          = records
    @storage_mode     = Paperclip::Attachment.default_options[:storage]
    @attachment_names = klass.attachment_definitions.keys
  end

  def delete
    remove_files
    batch.delete_all
  end

  def clear
    remove_files
    batch.update_all(nullified_attributes)
  end

  private

  def batch
    klass.where(id: records.map(&:id))
  end

  def remove_files
    keys = []

    logger.debug { "Preparing to delete attachments for #{records.size} records" }

    records.each do |record|
      @attachment_names.each do |attachment_name|
        attachment = record.public_send(attachment_name)
        styles     = BASE_STYLES | attachment.styles.keys

        next if attachment.blank?

        styles.each do |style|
          case @storage_mode
          when :s3
            logger.debug { "Adding #{attachment.path(style)} to batch for deletion" }
            keys << attachment.style_name_as_path(style)
          when :filesystem
            logger.debug { "Deleting #{attachment.path(style)}" }
            path = attachment.path(style)
            FileUtils.remove_file(path, true)

            begin
              FileUtils.rmdir(File.dirname(path), parents: true)
            rescue Errno::EEXIST, Errno::ENOTEMPTY, Errno::ENOENT, Errno::EINVAL, Errno::ENOTDIR, Errno::EACCES
              # Ignore failure to delete a directory, with the same ignored errors
              # as Paperclip
            end
          when :fog
            logger.debug { "Deleting #{attachment.path(style)}" }

            retries = 0
            begin
              attachment.send(:directory).files.new(key: attachment.path(style)).destroy
            rescue Fog::OpenStack::Storage::NotFound
              logger.debug "Will ignore because file is not found #{attachment.path(style)}"
            rescue => e
              retries += 1

              if retries < MAX_RETRY
                logger.debug "Retry #{retries}/#{MAX_RETRY} after #{e.message}"
                sleep 2**retries
                retry
              else
                logger.error "Batch deletion from fog failed after #{e.message}"
                raise e
              end
            end
          when :azure
            logger.debug { "Deleting #{attachment.path(style)}" }
            attachment.destroy
          end
        end
      end
    end

    return unless storage_mode == :s3

    # We can batch deletes over S3, but there is a limit of how many
    # objects can be processed at once, so we have to potentially
    # separate them into multiple calls.

    retries = 0
    keys.each_slice(LIMIT) do |keys_slice|
      logger.debug { "Deleting #{keys_slice.size} objects" }

      bucket.delete_objects(delete: {
        objects: keys_slice.map { |key| { key: key } },
        quiet: true,
      })
    rescue => e
      retries += 1

      if retries < MAX_RETRY
        logger.debug "Retry #{retries}/#{MAX_RETRY} after #{e.message}"
        sleep 2**retries
        retry
      else
        logger.error "Batch deletion from S3 failed after #{e.message}"
        raise e
      end
    end
  end

  def bucket
    @bucket ||= records.first.public_send(@attachment_names.first).s3_bucket
  end

  def nullified_attributes
    @attachment_names.flat_map { |attachment_name| NULLABLE_ATTRIBUTES.map { |attribute| "#{attachment_name}_#{attribute}" } & klass.column_names }.index_with(nil)
  end

  def logger
    Rails.logger
  end
end