From 5a8c651e8f0252c7135042e79396f782361302d9 Mon Sep 17 00:00:00 2001 From: Christian Schmidt Date: Fri, 3 Mar 2023 21:06:31 +0100 Subject: [PATCH] Only offer translation for supported languages (#23879) --- .rubocop.yml | 4 + .../mastodon/components/status_content.jsx | 4 +- app/javascript/mastodon/initial_state.js | 2 - app/lib/translation_service.rb | 4 + app/lib/translation_service/deepl.rb | 46 +++++--- .../translation_service/libre_translate.rb | 38 +++++-- app/models/status.rb | 10 ++ app/serializers/initial_state_serializer.rb | 1 - app/serializers/rest/status_serializer.rb | 6 +- app/services/translate_status_service.rb | 2 +- spec/lib/translation_service/deepl_spec.rb | 100 ++++++++++++++++++ .../libre_translate_spec.rb | 71 +++++++++++++ spec/models/status_spec.rb | 79 ++++++++++++++ 13 files changed, 336 insertions(+), 31 deletions(-) create mode 100644 spec/lib/translation_service/deepl_spec.rb create mode 100644 spec/lib/translation_service/libre_translate_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 27d778edf..0a41c54b9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -97,6 +97,10 @@ Rails/Exit: - 'lib/mastodon/cli_helper.rb' - 'lib/cli.rb' +RSpec/FilePath: + CustomTransform: + DeepL: deepl + RSpec/NotToNot: EnforcedStyle: to_not diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index a1c38171f..f9c9fe079 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -6,7 +6,7 @@ import { Link } from 'react-router-dom'; import classnames from 'classnames'; import PollContainer from 'mastodon/containers/poll_container'; import Icon from 'mastodon/components/icon'; -import { autoPlayGif, languages as preloadedLanguages, translationEnabled } from 'mastodon/initial_state'; +import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state'; const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top) @@ -220,7 +220,7 @@ class StatusContent extends React.PureComponent { const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden; const renderReadMore = this.props.onClick && status.get('collapsed'); - const renderTranslate = translationEnabled && this.context.identity.signedIn && this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && status.get('language') !== null && intl.locale !== status.get('language'); + const renderTranslate = this.props.onTranslate && status.get('translatable'); const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') }; const spoilerContent = { __html: status.get('spoilerHtml') }; diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js index d04c4a42d..919e0fc28 100644 --- a/app/javascript/mastodon/initial_state.js +++ b/app/javascript/mastodon/initial_state.js @@ -80,7 +80,6 @@ * @property {boolean} use_blurhash * @property {boolean=} use_pending_items * @property {string} version - * @property {boolean} translation_enabled */ /** @@ -132,7 +131,6 @@ export const unfollowModal = getMeta('unfollow_modal'); export const useBlurhash = getMeta('use_blurhash'); export const usePendingItems = getMeta('use_pending_items'); export const version = getMeta('version'); -export const translationEnabled = getMeta('translation_enabled'); export const languages = initialState?.languages; export const statusPageUrl = getMeta('status_page_url'); diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb index 285f30939..5ff93674a 100644 --- a/app/lib/translation_service.rb +++ b/app/lib/translation_service.rb @@ -21,6 +21,10 @@ class TranslationService ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? end + def supported?(_source_language, _target_language) + false + end + def translate(_text, _source_language, _target_language) raise NotImplementedError end diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index 151d33d90..deff95a1d 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService end def translate(text, source_language, target_language) - request(text, source_language, target_language).perform do |res| + form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' } + request(:post, '/v2/translate', form: form) do |res| + transform_response(res.body_with_limit) + end + end + + def supported?(source_language, target_language) + source_language.in?(languages('source')) && target_language.in?(languages('target')) + end + + private + + def languages(type) + Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do + request(:get, "/v2/languages?type=#{type}") do |res| + # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so + # they are supported but not returned by the API. + extra = type == 'source' ? [nil] : %w(en pt) + languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase } + + languages + extra + end + end + end + + def request(verb, path, **options) + req = Request.new(verb, "#{base_url}#{path}", **options) + req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") + req.perform do |res| case res.code when 429 raise TooManyRequestsError when 456 raise QuotaExceededError when 200...300 - transform_response(res.body_with_limit) + yield res else raise UnexpectedResponseError end end end - private - - def request(text, source_language, target_language) - req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }) - req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") - req - end - - def endpoint_url + def base_url if @plan == 'free' - 'https://api-free.deepl.com/v2/translate' + 'https://api-free.deepl.com' else - 'https://api.deepl.com/v2/translate' + 'https://api.deepl.com' end end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 4ebe21e45..743e4d77f 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService end def translate(text, source_language, target_language) - request(text, source_language, target_language).perform do |res| + body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) + request(:post, '/translate', body: body) do |res| + transform_response(res.body_with_limit, source_language) + end + end + + def supported?(source_language, target_language) + languages.key?(source_language) && languages[source_language].include?(target_language) + end + + private + + def languages + Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do + request(:get, '/languages') do |res| + languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] } + languages[nil] = languages.values.flatten.uniq + languages + end + end + end + + def request(verb, path, **options) + req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options) + req.add_headers('Content-Type': 'application/json') + req.perform do |res| case res.code when 429 raise TooManyRequestsError when 403 raise QuotaExceededError when 200...300 - transform_response(res.body_with_limit, source_language) + yield res else raise UnexpectedResponseError end end end - private - - def request(text, source_language, target_language) - body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) - req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true) - req.add_headers('Content-Type': 'application/json') - req - end - def transform_response(str, source_language) json = Oj.load(str, mode: :strict) diff --git a/app/models/status.rb b/app/models/status.rb index e7ea191a8..dd7ac2edb 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -232,6 +232,16 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end + def translatable? + translate_target_locale = I18n.locale.to_s.split(/[_-]/).first + + distributable? && + content.present? && + language != translate_target_locale && + TranslationService.configured? && + TranslationService.configured.supported?(language, translate_target_locale) + end + alias sign? distributable? def with_media? diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 7905444e9..769ba653e 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -30,7 +30,6 @@ class InitialStateSerializer < ActiveModel::Serializer timeline_preview: Setting.timeline_preview, activity_api_enabled: Setting.activity_api_enabled, single_user_mode: Rails.configuration.x.single_user_mode, - translation_enabled: TranslationService.configured?, trends_as_landing_page: Setting.trends_as_landing_page, status_page_url: Setting.status_page_url, } diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index e0b8f32a6..a422f5b25 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer include FormattingHelper attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id, - :sensitive, :spoiler_text, :visibility, :language, + :sensitive, :spoiler_text, :visibility, :language, :translatable, :uri, :url, :replies_count, :reblogs_count, :favourites_count, :edited_at @@ -50,6 +50,10 @@ class REST::StatusSerializer < ActiveModel::Serializer object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id) end + def translatable + current_user? && object.translatable? + end + def visibility # This visibility is masked behind "private" # to avoid API changes because there are no diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index 539a0d9db..92d8b62a0 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -6,7 +6,7 @@ class TranslateStatusService < BaseService include FormattingHelper def call(status, target_language) - raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility? + raise Mastodon::NotPermittedError unless status.translatable? @status = status @content = status_content_format(@status) diff --git a/spec/lib/translation_service/deepl_spec.rb b/spec/lib/translation_service/deepl_spec.rb new file mode 100644 index 000000000..aa2473186 --- /dev/null +++ b/spec/lib/translation_service/deepl_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TranslationService::DeepL do + subject(:service) { described_class.new(plan, 'my-api-key') } + + let(:plan) { 'advanced' } + + before do + stub_request(:get, 'https://api.deepl.com/v2/languages?type=source').to_return( + body: '[{"language":"EN","name":"English"},{"language":"UK","name":"Ukrainian"}]' + ) + stub_request(:get, 'https://api.deepl.com/v2/languages?type=target').to_return( + body: '[{"language":"EN-GB","name":"English (British)"},{"language":"ZH","name":"Chinese"}]' + ) + end + + describe '#supported?' do + it 'supports included languages as source and target languages' do + expect(service.supported?('uk', 'en')).to be true + end + + it 'supports auto-detecting source language' do + expect(service.supported?(nil, 'en')).to be true + end + + it 'supports "en" and "pt" as target languages though not included in language list' do + expect(service.supported?('uk', 'en')).to be true + expect(service.supported?('uk', 'pt')).to be true + end + + it 'does not support non-included language as target language' do + expect(service.supported?('uk', 'nl')).to be false + end + + it 'does not support non-included language as source language' do + expect(service.supported?('da', 'en')).to be false + end + end + + describe '#translate' do + it 'returns translation with specified source language' do + stub_request(:post, 'https://api.deepl.com/v2/translate') + .with(body: 'text=Hasta+la+vista&source_lang=ES&target_lang=en&tag_handling=html') + .to_return(body: '{"translations":[{"detected_source_language":"ES","text":"See you soon"}]}') + + translation = service.translate('Hasta la vista', 'es', 'en') + expect(translation.detected_source_language).to eq 'es' + expect(translation.provider).to eq 'DeepL.com' + expect(translation.text).to eq 'See you soon' + end + + it 'returns translation with auto-detected source language' do + stub_request(:post, 'https://api.deepl.com/v2/translate') + .with(body: 'text=Guten+Tag&source_lang&target_lang=en&tag_handling=html') + .to_return(body: '{"translations":[{"detected_source_language":"DE","text":"Good Morning"}]}') + + translation = service.translate('Guten Tag', nil, 'en') + expect(translation.detected_source_language).to eq 'de' + expect(translation.provider).to eq 'DeepL.com' + expect(translation.text).to eq 'Good Morning' + end + end + + describe '#languages?' do + it 'returns source languages' do + expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil] + end + + it 'returns target languages' do + expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt) + end + end + + describe '#request' do + before do + stub_request(:any, //) + # rubocop:disable Lint/EmptyBlock + service.send(:request, :get, '/v2/languages') { |res| } + # rubocop:enable Lint/EmptyBlock + end + + it 'uses paid plan base URL' do + expect(a_request(:get, 'https://api.deepl.com/v2/languages')).to have_been_made.once + end + + context 'with free plan' do + let(:plan) { 'free' } + + it 'uses free plan base URL' do + expect(a_request(:get, 'https://api-free.deepl.com/v2/languages')).to have_been_made.once + end + end + + it 'sends API key' do + expect(a_request(:get, 'https://api.deepl.com/v2/languages').with(headers: { Authorization: 'DeepL-Auth-Key my-api-key' })).to have_been_made.once + end + end +end diff --git a/spec/lib/translation_service/libre_translate_spec.rb b/spec/lib/translation_service/libre_translate_spec.rb new file mode 100644 index 000000000..a6cb01884 --- /dev/null +++ b/spec/lib/translation_service/libre_translate_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TranslationService::LibreTranslate do + subject(:service) { described_class.new('https://libretranslate.example.com', 'my-api-key') } + + before do + stub_request(:get, 'https://libretranslate.example.com/languages').to_return( + body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]' + ) + end + + describe '#supported?' do + it 'supports included language pair' do + expect(service.supported?('en', 'de')).to be true + end + + it 'does not support reversed language pair' do + expect(service.supported?('de', 'en')).to be false + end + + it 'supports auto-detecting source language' do + expect(service.supported?(nil, 'de')).to be true + end + + it 'does not support auto-detecting for unsupported target language' do + expect(service.supported?(nil, 'pt')).to be false + end + end + + describe '#languages' do + subject(:languages) { service.send(:languages) } + + it 'includes supported source languages' do + expect(languages.keys).to eq ['en', 'da', nil] + end + + it 'includes supported target languages for source language' do + expect(languages['en']).to eq %w(de es) + end + + it 'includes supported target languages for auto-detected language' do + expect(languages[nil]).to eq %w(de es en) + end + end + + describe '#translate' do + it 'returns translation with specified source language' do + stub_request(:post, 'https://libretranslate.example.com/translate') + .with(body: '{"q":"Hasta la vista","source":"es","target":"en","format":"html","api_key":"my-api-key"}') + .to_return(body: '{"translatedText": "See you"}') + + translation = service.translate('Hasta la vista', 'es', 'en') + expect(translation.detected_source_language).to eq 'es' + expect(translation.provider).to eq 'LibreTranslate' + expect(translation.text).to eq 'See you' + end + + it 'returns translation with auto-detected source language' do + stub_request(:post, 'https://libretranslate.example.com/translate') + .with(body: '{"q":"Guten Morgen","source":"auto","target":"en","format":"html","api_key":"my-api-key"}') + .to_return(body: '{"detectedLanguage":{"confidence":92,"language":"de"},"translatedText":"Good morning"}') + + translation = service.translate('Guten Morgen', nil, 'en') + expect(translation.detected_source_language).to be_nil + expect(translation.provider).to eq 'LibreTranslate' + expect(translation.text).to eq 'Good morning' + end + end +end diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index 1e58c6d0d..1f6cfc796 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -114,6 +114,85 @@ RSpec.describe Status, type: :model do end end + describe '#translatable?' do + before do + allow(TranslationService).to receive(:configured?).and_return(true) + allow(TranslationService).to receive(:configured).and_return(TranslationService.new) + allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true) + + subject.language = 'es' + subject.visibility = :public + end + + context 'all conditions are satisfied' do + it 'returns true' do + expect(subject.translatable?).to be true + end + end + + context 'translation service is not configured' do + it 'returns false' do + allow(TranslationService).to receive(:configured?).and_return(false) + allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError) + expect(subject.translatable?).to be false + end + end + + context 'status language is nil' do + it 'returns true' do + subject.language = nil + allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true) + expect(subject.translatable?).to be true + end + end + + context 'status language is same as default locale' do + it 'returns false' do + subject.language = I18n.locale + expect(subject.translatable?).to be false + end + end + + context 'status language is unsupported' do + it 'returns false' do + subject.language = 'af' + allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false) + expect(subject.translatable?).to be false + end + end + + context 'default locale is unsupported' do + it 'returns false' do + allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false) + I18n.with_locale('af') do + expect(subject.translatable?).to be false + end + end + end + + context 'default locale has region' do + it 'returns true' do + I18n.with_locale('en-GB') do + expect(subject.translatable?).to be true + end + end + end + + context 'status text is blank' do + it 'returns false' do + subject.text = ' ' + expect(subject.translatable?).to be false + end + end + + context 'status visiblity is hidden' do + it 'returns false' do + subject.visibility = 'limited' + expect(subject.translatable?).to be false + end + end + end + describe '#content' do it 'returns the text of the status if it is not a reblog' do expect(subject.content).to eql subject.text