From 5a8c651e8f0252c7135042e79396f782361302d9 Mon Sep 17 00:00:00 2001
From: Christian Schmidt <github@chsc.dk>
Date: Fri, 3 Mar 2023 21:06:31 +0100
Subject: [PATCH] Only offer translation for supported languages (#23879)

---
 .rubocop.yml                                  |   4 +
 .../mastodon/components/status_content.jsx    |   4 +-
 app/javascript/mastodon/initial_state.js      |   2 -
 app/lib/translation_service.rb                |   4 +
 app/lib/translation_service/deepl.rb          |  46 +++++---
 .../translation_service/libre_translate.rb    |  38 +++++--
 app/models/status.rb                          |  10 ++
 app/serializers/initial_state_serializer.rb   |   1 -
 app/serializers/rest/status_serializer.rb     |   6 +-
 app/services/translate_status_service.rb      |   2 +-
 spec/lib/translation_service/deepl_spec.rb    | 100 ++++++++++++++++++
 .../libre_translate_spec.rb                   |  71 +++++++++++++
 spec/models/status_spec.rb                    |  79 ++++++++++++++
 13 files changed, 336 insertions(+), 31 deletions(-)
 create mode 100644 spec/lib/translation_service/deepl_spec.rb
 create mode 100644 spec/lib/translation_service/libre_translate_spec.rb

diff --git a/.rubocop.yml b/.rubocop.yml
index 27d778edfb..0a41c54b90 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -97,6 +97,10 @@ Rails/Exit:
     - 'lib/mastodon/cli_helper.rb'
     - 'lib/cli.rb'
 
+RSpec/FilePath:
+  CustomTransform:
+    DeepL: deepl
+
 RSpec/NotToNot:
   EnforcedStyle: to_not
 
diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx
index a1c38171f2..f9c9fe0791 100644
--- a/app/javascript/mastodon/components/status_content.jsx
+++ b/app/javascript/mastodon/components/status_content.jsx
@@ -6,7 +6,7 @@ import { Link } from 'react-router-dom';
 import classnames from 'classnames';
 import PollContainer from 'mastodon/containers/poll_container';
 import Icon from 'mastodon/components/icon';
-import { autoPlayGif, languages as preloadedLanguages, translationEnabled } from 'mastodon/initial_state';
+import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state';
 
 const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top)
 
@@ -220,7 +220,7 @@ class StatusContent extends React.PureComponent {
 
     const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden;
     const renderReadMore = this.props.onClick && status.get('collapsed');
-    const renderTranslate = translationEnabled && this.context.identity.signedIn && this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && status.get('language') !== null && intl.locale !== status.get('language');
+    const renderTranslate = this.props.onTranslate && status.get('translatable');
 
     const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') };
     const spoilerContent = { __html: status.get('spoilerHtml') };
diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js
index d04c4a42d2..919e0fc282 100644
--- a/app/javascript/mastodon/initial_state.js
+++ b/app/javascript/mastodon/initial_state.js
@@ -80,7 +80,6 @@
  * @property {boolean} use_blurhash
  * @property {boolean=} use_pending_items
  * @property {string} version
- * @property {boolean} translation_enabled
  */
 
 /**
@@ -132,7 +131,6 @@ export const unfollowModal = getMeta('unfollow_modal');
 export const useBlurhash = getMeta('use_blurhash');
 export const usePendingItems = getMeta('use_pending_items');
 export const version = getMeta('version');
-export const translationEnabled = getMeta('translation_enabled');
 export const languages = initialState?.languages;
 export const statusPageUrl = getMeta('status_page_url');
 
diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb
index 285f309393..5ff93674a4 100644
--- a/app/lib/translation_service.rb
+++ b/app/lib/translation_service.rb
@@ -21,6 +21,10 @@ class TranslationService
     ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
   end
 
+  def supported?(_source_language, _target_language)
+    false
+  end
+
   def translate(_text, _source_language, _target_language)
     raise NotImplementedError
   end
diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb
index 151d33d909..deff95a1db 100644
--- a/app/lib/translation_service/deepl.rb
+++ b/app/lib/translation_service/deepl.rb
@@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
+    request(:post, '/v2/translate', form: form) do |res|
+      transform_response(res.body_with_limit)
+    end
+  end
+
+  def supported?(source_language, target_language)
+    source_language.in?(languages('source')) && target_language.in?(languages('target'))
+  end
+
+  private
+
+  def languages(type)
+    Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do
+      request(:get, "/v2/languages?type=#{type}") do |res|
+        # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
+        # they are supported but not returned by the API.
+        extra = type == 'source' ? [nil] : %w(en pt)
+        languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase }
+
+        languages + extra
+      end
+    end
+  end
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{base_url}#{path}", **options)
+    req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 456
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' })
-    req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
-    req
-  end
-
-  def endpoint_url
+  def base_url
     if @plan == 'free'
-      'https://api-free.deepl.com/v2/translate'
+      'https://api-free.deepl.com'
     else
-      'https://api.deepl.com/v2/translate'
+      'https://api.deepl.com'
     end
   end
 
diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb
index 4ebe21e454..743e4d77f7 100644
--- a/app/lib/translation_service/libre_translate.rb
+++ b/app/lib/translation_service/libre_translate.rb
@@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService
   end
 
   def translate(text, source_language, target_language)
-    request(text, source_language, target_language).perform do |res|
+    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
+    request(:post, '/translate', body: body) do |res|
+      transform_response(res.body_with_limit, source_language)
+    end
+  end
+
+  def supported?(source_language, target_language)
+    languages.key?(source_language) && languages[source_language].include?(target_language)
+  end
+
+  private
+
+  def languages
+    Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do
+      request(:get, '/languages') do |res|
+        languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] }
+        languages[nil] = languages.values.flatten.uniq
+        languages
+      end
+    end
+  end
+
+  def request(verb, path, **options)
+    req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
+    req.add_headers('Content-Type': 'application/json')
+    req.perform do |res|
       case res.code
       when 429
         raise TooManyRequestsError
       when 403
         raise QuotaExceededError
       when 200...300
-        transform_response(res.body_with_limit, source_language)
+        yield res
       else
         raise UnexpectedResponseError
       end
     end
   end
 
-  private
-
-  def request(text, source_language, target_language)
-    body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
-    req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true)
-    req.add_headers('Content-Type': 'application/json')
-    req
-  end
-
   def transform_response(str, source_language)
     json = Oj.load(str, mode: :strict)
 
diff --git a/app/models/status.rb b/app/models/status.rb
index e7ea191a80..dd7ac2edb1 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -232,6 +232,16 @@ class Status < ApplicationRecord
     public_visibility? || unlisted_visibility?
   end
 
+  def translatable?
+    translate_target_locale = I18n.locale.to_s.split(/[_-]/).first
+
+    distributable? &&
+      content.present? &&
+      language != translate_target_locale &&
+      TranslationService.configured? &&
+      TranslationService.configured.supported?(language, translate_target_locale)
+  end
+
   alias sign? distributable?
 
   def with_media?
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 7905444e98..769ba653ed 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -30,7 +30,6 @@ class InitialStateSerializer < ActiveModel::Serializer
       timeline_preview: Setting.timeline_preview,
       activity_api_enabled: Setting.activity_api_enabled,
       single_user_mode: Rails.configuration.x.single_user_mode,
-      translation_enabled: TranslationService.configured?,
       trends_as_landing_page: Setting.trends_as_landing_page,
       status_page_url: Setting.status_page_url,
     }
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index e0b8f32a68..a422f5b258 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   include FormattingHelper
 
   attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id,
-             :sensitive, :spoiler_text, :visibility, :language,
+             :sensitive, :spoiler_text, :visibility, :language, :translatable,
              :uri, :url, :replies_count, :reblogs_count,
              :favourites_count, :edited_at
 
@@ -50,6 +50,10 @@ class REST::StatusSerializer < ActiveModel::Serializer
     object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id)
   end
 
+  def translatable
+    current_user? && object.translatable?
+  end
+
   def visibility
     # This visibility is masked behind "private"
     # to avoid API changes because there are no
diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb
index 539a0d9db5..92d8b62a05 100644
--- a/app/services/translate_status_service.rb
+++ b/app/services/translate_status_service.rb
@@ -6,7 +6,7 @@ class TranslateStatusService < BaseService
   include FormattingHelper
 
   def call(status, target_language)
-    raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility?
+    raise Mastodon::NotPermittedError unless status.translatable?
 
     @status = status
     @content = status_content_format(@status)
diff --git a/spec/lib/translation_service/deepl_spec.rb b/spec/lib/translation_service/deepl_spec.rb
new file mode 100644
index 0000000000..aa24731860
--- /dev/null
+++ b/spec/lib/translation_service/deepl_spec.rb
@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe TranslationService::DeepL do
+  subject(:service) { described_class.new(plan, 'my-api-key') }
+
+  let(:plan) { 'advanced' }
+
+  before do
+    stub_request(:get, 'https://api.deepl.com/v2/languages?type=source').to_return(
+      body: '[{"language":"EN","name":"English"},{"language":"UK","name":"Ukrainian"}]'
+    )
+    stub_request(:get, 'https://api.deepl.com/v2/languages?type=target').to_return(
+      body: '[{"language":"EN-GB","name":"English (British)"},{"language":"ZH","name":"Chinese"}]'
+    )
+  end
+
+  describe '#supported?' do
+    it 'supports included languages as source and target languages' do
+      expect(service.supported?('uk', 'en')).to be true
+    end
+
+    it 'supports auto-detecting source language' do
+      expect(service.supported?(nil, 'en')).to be true
+    end
+
+    it 'supports "en" and "pt" as target languages though not included in language list' do
+      expect(service.supported?('uk', 'en')).to be true
+      expect(service.supported?('uk', 'pt')).to be true
+    end
+
+    it 'does not support non-included language as target language' do
+      expect(service.supported?('uk', 'nl')).to be false
+    end
+
+    it 'does not support non-included language as source language' do
+      expect(service.supported?('da', 'en')).to be false
+    end
+  end
+
+  describe '#translate' do
+    it 'returns translation with specified source language' do
+      stub_request(:post, 'https://api.deepl.com/v2/translate')
+        .with(body: 'text=Hasta+la+vista&source_lang=ES&target_lang=en&tag_handling=html')
+        .to_return(body: '{"translations":[{"detected_source_language":"ES","text":"See you soon"}]}')
+
+      translation = service.translate('Hasta la vista', 'es', 'en')
+      expect(translation.detected_source_language).to eq 'es'
+      expect(translation.provider).to eq 'DeepL.com'
+      expect(translation.text).to eq 'See you soon'
+    end
+
+    it 'returns translation with auto-detected source language' do
+      stub_request(:post, 'https://api.deepl.com/v2/translate')
+        .with(body: 'text=Guten+Tag&source_lang&target_lang=en&tag_handling=html')
+        .to_return(body: '{"translations":[{"detected_source_language":"DE","text":"Good Morning"}]}')
+
+      translation = service.translate('Guten Tag', nil, 'en')
+      expect(translation.detected_source_language).to eq 'de'
+      expect(translation.provider).to eq 'DeepL.com'
+      expect(translation.text).to eq 'Good Morning'
+    end
+  end
+
+  describe '#languages?' do
+    it 'returns source languages' do
+      expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil]
+    end
+
+    it 'returns target languages' do
+      expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt)
+    end
+  end
+
+  describe '#request' do
+    before do
+      stub_request(:any, //)
+      # rubocop:disable Lint/EmptyBlock
+      service.send(:request, :get, '/v2/languages') { |res| }
+      # rubocop:enable Lint/EmptyBlock
+    end
+
+    it 'uses paid plan base URL' do
+      expect(a_request(:get, 'https://api.deepl.com/v2/languages')).to have_been_made.once
+    end
+
+    context 'with free plan' do
+      let(:plan) { 'free' }
+
+      it 'uses free plan base URL' do
+        expect(a_request(:get, 'https://api-free.deepl.com/v2/languages')).to have_been_made.once
+      end
+    end
+
+    it 'sends API key' do
+      expect(a_request(:get, 'https://api.deepl.com/v2/languages').with(headers: { Authorization: 'DeepL-Auth-Key my-api-key' })).to have_been_made.once
+    end
+  end
+end
diff --git a/spec/lib/translation_service/libre_translate_spec.rb b/spec/lib/translation_service/libre_translate_spec.rb
new file mode 100644
index 0000000000..a6cb01884a
--- /dev/null
+++ b/spec/lib/translation_service/libre_translate_spec.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe TranslationService::LibreTranslate do
+  subject(:service) { described_class.new('https://libretranslate.example.com', 'my-api-key') }
+
+  before do
+    stub_request(:get, 'https://libretranslate.example.com/languages').to_return(
+      body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]'
+    )
+  end
+
+  describe '#supported?' do
+    it 'supports included language pair' do
+      expect(service.supported?('en', 'de')).to be true
+    end
+
+    it 'does not support reversed language pair' do
+      expect(service.supported?('de', 'en')).to be false
+    end
+
+    it 'supports auto-detecting source language' do
+      expect(service.supported?(nil, 'de')).to be true
+    end
+
+    it 'does not support auto-detecting for unsupported target language' do
+      expect(service.supported?(nil, 'pt')).to be false
+    end
+  end
+
+  describe '#languages' do
+    subject(:languages) { service.send(:languages) }
+
+    it 'includes supported source languages' do
+      expect(languages.keys).to eq ['en', 'da', nil]
+    end
+
+    it 'includes supported target languages for source language' do
+      expect(languages['en']).to eq %w(de es)
+    end
+
+    it 'includes supported target languages for auto-detected language' do
+      expect(languages[nil]).to eq %w(de es en)
+    end
+  end
+
+  describe '#translate' do
+    it 'returns translation with specified source language' do
+      stub_request(:post, 'https://libretranslate.example.com/translate')
+        .with(body: '{"q":"Hasta la vista","source":"es","target":"en","format":"html","api_key":"my-api-key"}')
+        .to_return(body: '{"translatedText": "See you"}')
+
+      translation = service.translate('Hasta la vista', 'es', 'en')
+      expect(translation.detected_source_language).to eq 'es'
+      expect(translation.provider).to eq 'LibreTranslate'
+      expect(translation.text).to eq 'See you'
+    end
+
+    it 'returns translation with auto-detected source language' do
+      stub_request(:post, 'https://libretranslate.example.com/translate')
+        .with(body: '{"q":"Guten Morgen","source":"auto","target":"en","format":"html","api_key":"my-api-key"}')
+        .to_return(body: '{"detectedLanguage":{"confidence":92,"language":"de"},"translatedText":"Good morning"}')
+
+      translation = service.translate('Guten Morgen', nil, 'en')
+      expect(translation.detected_source_language).to be_nil
+      expect(translation.provider).to eq 'LibreTranslate'
+      expect(translation.text).to eq 'Good morning'
+    end
+  end
+end
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index 1e58c6d0d1..1f6cfc7967 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -114,6 +114,85 @@ RSpec.describe Status, type: :model do
     end
   end
 
+  describe '#translatable?' do
+    before do
+      allow(TranslationService).to receive(:configured?).and_return(true)
+      allow(TranslationService).to receive(:configured).and_return(TranslationService.new)
+      allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true)
+
+      subject.language = 'es'
+      subject.visibility = :public
+    end
+
+    context 'all conditions are satisfied' do
+      it 'returns true' do
+        expect(subject.translatable?).to be true
+      end
+    end
+
+    context 'translation service is not configured' do
+      it 'returns false' do
+        allow(TranslationService).to receive(:configured?).and_return(false)
+        allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError)
+        expect(subject.translatable?).to be false
+      end
+    end
+
+    context 'status language is nil' do
+      it 'returns true' do
+        subject.language = nil
+        allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true)
+        expect(subject.translatable?).to be true
+      end
+    end
+
+    context 'status language is same as default locale' do
+      it 'returns false' do
+        subject.language = I18n.locale
+        expect(subject.translatable?).to be false
+      end
+    end
+
+    context 'status language is unsupported' do
+      it 'returns false' do
+        subject.language = 'af'
+        allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false)
+        expect(subject.translatable?).to be false
+      end
+    end
+
+    context 'default locale is unsupported' do
+      it 'returns false' do
+        allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false)
+        I18n.with_locale('af') do
+          expect(subject.translatable?).to be false
+        end
+      end
+    end
+
+    context 'default locale has region' do
+      it 'returns true' do
+        I18n.with_locale('en-GB') do
+          expect(subject.translatable?).to be true
+        end
+      end
+    end
+
+    context 'status text is blank' do
+      it 'returns false' do
+        subject.text = ' '
+        expect(subject.translatable?).to be false
+      end
+    end
+
+    context 'status visiblity is hidden' do
+      it 'returns false' do
+        subject.visibility = 'limited'
+        expect(subject.translatable?).to be false
+      end
+    end
+  end
+
   describe '#content' do
     it 'returns the text of the status if it is not a reblog' do
       expect(subject.content).to eql subject.text