diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake
index d1de17b7c0..86c358a94e 100644
--- a/lib/tasks/repo.rake
+++ b/lib/tasks/repo.rake
@@ -1,27 +1,34 @@
 # frozen_string_literal: true
 
+REPOSITORY_NAME = 'tootsuite/mastodon'
+
 namespace :repo do
   desc 'Generate the AUTHORS.md file'
   task :authors do
     file = File.open(Rails.root.join('AUTHORS.md'), 'w')
+
     file << <<~HEADER
       Authors
       =======
 
-      Mastodon is available on [GitHub](https://github.com/tootsuite/mastodon)
+      Mastodon is available on [GitHub](https://github.com/#{REPOSITORY_NAME})
       and provided thanks to the work of the following contributors:
 
     HEADER
 
-    url = 'https://api.github.com/repos/tootsuite/mastodon/contributors?anon=1'
+    url = "https://api.github.com/repos/#{REPOSITORY_NAME}/contributors?anon=1"
+
     HttpLog.config.compact_log = true
+
     while url.present?
-      response = HTTP.get(url)
+      response     = HTTP.get(url)
       contributors = Oj.load(response.body)
+
       contributors.each do |c|
         file << "* [#{c['login']}](#{c['html_url']})\n" if c['login']
         file << "* [#{c['name']}](mailto:#{c['email']})\n" if c['name']
       end
+
       url = LinkHeader.parse(response.headers['Link']).find_link(%w(rel next))&.href
     end
 
@@ -47,7 +54,7 @@ namespace :repo do
               response = nil
 
               loop do
-                response = HTTP.headers('Authorization' => "token #{ENV['GITHUB_API_TOKEN']}").get("https://api.github.com/repos/tootsuite/mastodon/pulls/#{pull_request_number}")
+                response = HTTP.headers('Authorization' => "token #{ENV['GITHUB_API_TOKEN']}").get("https://api.github.com/repos/#{REPOSITORY_NAME}/pulls/#{pull_request_number}")
 
                 if response.code == 403
                   sleep_for = (response.headers['X-RateLimit-Reset'].to_i - Time.now.to_i).abs
@@ -83,12 +90,46 @@ namespace :repo do
     missing_yaml_files = I18n.available_locales.reject { |locale| File.exist?(Rails.root.join('config', 'locales', "#{locale}.yml")) }
     missing_json_files = I18n.available_locales.reject { |locale| File.exist?(Rails.root.join('app', 'javascript', 'mastodon', 'locales', "#{locale}.json")) }
 
-    if missing_json_files.empty? && missing_yaml_files.empty?
-      puts pastel.green('OK')
-    else
-      puts pastel.red("Missing YAML files: #{pastel.bold(missing_yaml_files.join(', '))}") unless missing_yaml_files.empty?
-      puts pastel.red("Missing JSON files: #{pastel.bold(missing_json_files.join(', '))}") unless missing_json_files.empty?
+    locales_in_files = Dir[Rails.root.join('config', 'locales', '*.yml')].map do |path|
+      file_name = File.basename(path)
+      file_name.gsub(/\A(doorkeeper|devise|activerecord|simple_form)\./, '').gsub(/\.yml\z/, '').to_sym
+    end.uniq.compact
+
+    missing_available_locales = locales_in_files - I18n.available_locales
+    missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) }
+
+    critical = false
+
+    unless missing_json_files.empty?
+      critical = true
+
+      puts pastel.red("You are missing JSON files for these locales: #{pastel.bold(missing_json_files.join(', '))}")
+      puts pastel.red('This will lead to runtime errors for users who have selected those locales')
+      puts pastel.red("Add the missing files or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
+    end
+
+    unless missing_yaml_files.empty?
+      critical = true
+
+      puts pastel.red("You are missing YAML files for these locales: #{pastel.bold(missing_yaml_files.join(', '))}")
+      puts pastel.red('This will lead to runtime errors for users who have selected those locales')
+      puts pastel.red("Add the missing files or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
+    end
+
+    unless missing_available_locales.empty?
+      puts pastel.yellow("You have locale files that are not enabled: #{pastel.bold(missing_available_locales.join(', '))}")
+      puts pastel.yellow("Add them to #{pastel.bold('I18n.available_locales')} in config/application.rb or remove them")
+    end
+
+    unless missing_locale_names.empty?
+      puts pastel.yellow("You are missing human-readable names for these locales: #{pastel.bold(missing_locale_names.join(', '))}")
+      puts pastel.yellow("Add them to #{pastel.bold('HUMAN_LOCALES')} in app/helpers/settings_helper.rb or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
+    end
+
+    if critical
       exit(1)
+    else
+      puts pastel.green('OK')
     end
   end
 end