Change tootctl media remove-orphans to work for all classes (#13316)

Change `tootctl media lookup` to not use an interactive prompt

app/lib/activitypub/tag_manager.rb

@@ -15,6 +15,8 @@ class ActivityPub::TagManager
   def url_for(target)
     return target.url if target.respond_to?(:local?) && !target.local?
 
+    return unless target.respond_to?(:object_type)
+
     case target.object_type
     when :person
       target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)

app/models/media_attachment.rb

@@ -184,19 +184,6 @@ class MediaAttachment < ApplicationRecord
     audio? || video?
   end
 
-  def variant?(other_file_name)
-    return true if file_file_name == other_file_name
-    return false if file_file_name.nil?
-
-    formats = file.styles.values.map(&:format).compact
-
-    return false if formats.empty?
-
-    extension = File.extname(other_file_name)
-
-    formats.include?(extension.delete('.')) && File.basename(other_file_name, extension) == File.basename(file_file_name, File.extname(file_file_name))
-  end
-
   def to_param
     shortcode
   end

lib/mastodon/media_cli.rb

@@ -45,6 +45,7 @@ module Mastodon
     end
 
     option :start_after
+    option :prefix
     option :dry_run, type: :boolean, default: false
     desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments'
     long_desc <<~LONG_DESC
@@ -58,6 +59,7 @@ module Mastodon
       reclaimed_bytes = 0
       removed         = 0
       dry_run         = options[:dry_run] ? ' (DRY RUN)' : ''
+      prefix          = options[:prefix]
 
       case Paperclip::Attachment.default_options[:storage]
       when :s3
@@ -69,7 +71,7 @@ module Mastodon
         loop do
           objects = begin
             begin
-              bucket.objects(start_after: last_key, prefix: 'media_attachments/files/').limit(1000).map { |x| x }
+              bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
             rescue => e
               progress.log(pastel.red("Error fetching list of files: #{e}"))
               progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
@@ -79,16 +81,21 @@ module Mastodon
 
           break if objects.empty?
 
-          last_key        = objects.last.key
+          last_key   = objects.last.key
-          attachments_map = MediaAttachment.where(id: objects.map { |object| object.key.split('/')[2..-2].join.to_i }).each_with_object({}) { |attachment, map| map[attachment.id] = attachment }
+          record_map = preload_records_from_mixed_objects(objects)
 
           objects.each do |object|
-            attachment_id = object.key.split('/')[2..-2].join.to_i
+            path_segments   = object.key.split('/')
-            filename      = object.key.split('/').last
+            model_name      = path_segments.first.classify
+            attachment_name = path_segments[1].singularize
+            record_id       = path_segments[2..-2].join.to_i
+            file_name       = path_segments.last
+            record          = record_map.dig(model_name, record_id)
+            attachment      = record&.public_send(attachment_name)
 
             progress.increment
 
-            next unless attachments_map[attachment_id].nil? || !attachments_map[attachment_id].variant?(filename)
+            next unless attachment.blank? || !attachment.variant?(file_name)
 
             begin
               object.delete unless options[:dry_run]
@@ -110,17 +117,24 @@ module Mastodon
 
         root_path = ENV.fetch('RAILS_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
 
-        Find.find(File.join(root_path, 'media_attachments', 'files')) do |path|
+        Find.find(File.join(*[root_path, prefix].compact)) do |path|
           next if File.directory?(path)
 
-          key           = path.gsub("#{root_path}#{File::SEPARATOR}", '')
+          key             = path.gsub("#{root_path}#{File::SEPARATOR}", '')
-          attachment_id = key.split(File::SEPARATOR)[2..-2].join.to_i
+          path_segments   = key.split(File::SEPARATOR)
-          filename      = key.split(File::SEPARATOR).last
+          model_name      = path_segments.first.classify
-          attachment    = MediaAttachment.find_by(id: attachment_id)
+          record_id       = path_segments[2..-2].join.to_i
+          attachment_name = path_segments[1].singularize
+          file_name       = path_segments.last
+
+          next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
+
+          record     = model_name.constantize.find_by(id: record_id)
+          attachment = record&.public_send(attachment_name)
 
           progress.increment
 
-          next unless attachment.nil? || !attachment.variant?(filename)
+          next unless attachment.blank? || !attachment.variant?(file_name)
 
           begin
             size = File.size(path)
@@ -213,25 +227,66 @@ module Mastodon
       say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
     end
 
-    desc 'lookup', 'Lookup where media is displayed by passing a media URL'
+    desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
-    def lookup
+    def lookup(url)
-      prompt = TTY::Prompt.new
+      path          = Addressable::URI.parse(url).path
+      path_segments = path.split('/')[2..-1]
+      model_name    = path_segments.first.classify
+      record_id     = path_segments[2..-2].join.to_i
 
-      url = prompt.ask('Please enter a URL to the media to lookup:', required: true)
+      unless PRELOAD_MODEL_WHITELIST.include?(model_name)
+        say("Cannot find corresponding model: #{model_name}", :red)
+        exit(1)
+      end
 
-      attachment_id = url
+      record = model_name.constantize.find_by(id: record_id)
-                      .split('/')[0..-2]
+      record = record.status if record.respond_to?(:status)
-                      .grep(/\A\d+\z/)
-                      .join('')
 
-      if url.split('/')[0..-2].include? 'media_attachments'
+      unless record
-        model = MediaAttachment.find(attachment_id).status
+        say('Cannot find corresponding record', :red)
-        prompt.say(ActivityPub::TagManager.instance.url_for(model))
+        exit(1)
-      elsif url.split('/')[0..-2].include? 'accounts'
+      end
-        model = Account.find(attachment_id)
+
-        prompt.say(ActivityPub::TagManager.instance.url_for(model))
+      display_url = ActivityPub::TagManager.instance.url_for(record)
-      else
+
-        prompt.say('Not found')
+      if display_url.blank?
+        say('No public URL for this type of record', :red)
+        exit(1)
+      end
+
+      say(display_url, :blue)
+    rescue Addressable::URI::InvalidURIError
+      say('Invalid URL', :red)
+      exit(1)
+    end
+
+    private
+
+    PRELOAD_MODEL_WHITELIST = %w(
+      Account
+      Backup
+      CustomEmoji
+      Import
+      MediaAttachment
+      PreviewCard
+      SiteUpload
+    ).freeze
+
+    def preload_records_from_mixed_objects(objects)
+      preload_map = Hash.new { |hash, key| hash[key] = [] }
+
+      objects.map do |object|
+        segments   = object.key.split('/').first
+        model_name = segments.first.classify
+        record_id  = segments[2..-2].join.to_i
+
+        next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
+
+        preload_map[model_name] << record_id
+      end
+
+      preload_map.each_with_object({}) do |(model_name, record_ids), model_map|
+        model_map[model_name] = model_name.constantize.where(id: record_ids).each_with_object({}) { |record, record_map| record_map[record.id] = record }
       end
     end
   end

lib/paperclip/attachment_extensions.rb

@@ -24,6 +24,19 @@ module Paperclip
         flush_deletes
       end
     end
+
+    def variant?(other_filename)
+      return true  if original_filename == other_filename
+      return false if original_filename.nil?
+
+      formats = styles.values.map(&:format).compact
+
+      return false if formats.empty?
+
+      other_extension = File.extname(other_filename)
+
+      formats.include?(other_extension.delete('.')) && File.basename(other_filename, other_extension) == File.basename(original_filename, File.extname(original_filename))
+    end
   end
 end