Add administrative webhooks (#18510)

* Add administrative webhooks

* Fix error when webhook is deleted before delivery worker runs

app/controllers/admin/webhooks/secrets_controller.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Admin
+  class Webhooks::SecretsController < BaseController
+    before_action :set_webhook
+
+    def rotate
+      authorize @webhook, :rotate_secret?
+      @webhook.rotate_secret!
+      redirect_to admin_webhook_path(@webhook)
+    end
+
+    private
+
+    def set_webhook
+      @webhook = Webhook.find(params[:webhook_id])
+    end
+  end
+end

app/controllers/admin/webhooks_controller.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Admin
+  class WebhooksController < BaseController
+    before_action :set_webhook, except: [:index, :new, :create]
+
+    def index
+      authorize :webhook, :index?
+
+      @webhooks = Webhook.page(params[:page])
+    end
+
+    def new
+      authorize :webhook, :create?
+
+      @webhook = Webhook.new
+    end
+
+    def create
+      authorize :webhook, :create?
+
+      @webhook = Webhook.new(resource_params)
+
+      if @webhook.save
+        redirect_to admin_webhook_path(@webhook)
+      else
+        render :new
+      end
+    end
+
+    def show
+      authorize @webhook, :show?
+    end
+
+    def edit
+      authorize @webhook, :update?
+    end
+
+    def update
+      authorize @webhook, :update?
+
+      if @webhook.update(resource_params)
+        redirect_to admin_webhook_path(@webhook)
+      else
+        render :show
+      end
+    end
+
+    def enable
+      authorize @webhook, :enable?
+      @webhook.enable!
+      redirect_to admin_webhook_path(@webhook)
+    end
+
+    def disable
+      authorize @webhook, :disable?
+      @webhook.disable!
+      redirect_to admin_webhook_path(@webhook)
+    end
+
+    def destroy
+      authorize @webhook, :destroy?
+      @webhook.destroy!
+      redirect_to admin_webhooks_path
+    end
+
+    private
+
+    def set_webhook
+      @webhook = Webhook.find(params[:id])
+    end
+
+    def resource_params
+      params.require(:webhook).permit(:url, events: [])
+    end
+  end
+end

app/javascript/styles/mastodon/admin.scss

@@ -203,6 +203,14 @@ $content-width: 840px;
         }
       }
 
+      h2 small {
+        font-size: 12px;
+        display: block;
+        font-weight: 500;
+        color: $darker-text-color;
+        line-height: 18px;
+      }
+
       @media screen and (max-width: $no-columns-breakpoint) {
         border-bottom: 0;
         padding-bottom: 0;

app/models/admin/action_log.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 # == Schema Information
 #
 # Table name: admin_action_logs

app/models/report.rb

@@ -55,6 +55,8 @@ class Report < ApplicationRecord
 
   before_validation :set_uri, only: :create
 
+  after_create_commit :trigger_webhooks
+
   def object_type
     :flag
   end
@@ -143,4 +145,8 @@ class Report < ApplicationRecord
 
     errors.add(:rule_ids, I18n.t('reports.errors.invalid_rules')) unless rules.size == rule_ids&.size
   end
+
+  def trigger_webhooks
+    TriggerWebhookWorker.perform_async('report.created', 'Report', id)
+  end
 end

app/models/user.rb

@@ -37,7 +37,6 @@
 #  sign_in_token_sent_at     :datetime
 #  webauthn_id               :string
 #  sign_up_ip                :inet
-#  skip_sign_in_token        :boolean
 #
 
 class User < ApplicationRecord
@@ -120,6 +119,7 @@ class User < ApplicationRecord
   before_validation :sanitize_languages
   before_create :set_approved
   after_commit :send_pending_devise_notifications
+  after_create_commit :trigger_webhooks
 
   # This avoids a deprecation warning from Rails 5.1
   # It seems possible that a future release of devise-two-factor will
@@ -182,7 +182,9 @@ class User < ApplicationRecord
   end
 
   def update_sign_in!(new_sign_in: false)
-    old_current, new_current = current_sign_in_at, Time.now.utc
+    old_current = current_sign_in_at
+    new_current = Time.now.utc
+
     self.last_sign_in_at     = old_current || new_current
     self.current_sign_in_at  = new_current
 
@@ -472,4 +474,8 @@ class User < ApplicationRecord
   def invite_text_required?
     Setting.require_invite_text && !invited? && !external? && !bypass_invite_request_check?
   end
+
+  def trigger_webhooks
+    TriggerWebhookWorker.perform_async('account.created', 'Account', account_id)
+  end
 end

app/models/webhook.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: webhooks
+#
+#  id         :bigint(8)        not null, primary key
+#  url        :string           not null
+#  events     :string           default([]), not null, is an Array
+#  secret     :string           default(""), not null
+#  enabled    :boolean          default(TRUE), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class Webhook < ApplicationRecord
+  EVENTS = %w(
+    account.created
+    report.created
+  ).freeze
+
+  scope :enabled, -> { where(enabled: true) }
+
+  validates :url, presence: true, url: true
+  validates :secret, presence: true, length: { minimum: 12 }
+  validates :events, presence: true
+
+  validate :validate_events
+
+  before_validation :strip_events
+  before_validation :generate_secret
+
+  def rotate_secret!
+    update!(secret: SecureRandom.hex(20))
+  end
+
+  def enable!
+    update!(enabled: true)
+  end
+
+  def disable!
+    update!(enabled: false)
+  end
+
+  private
+
+  def validate_events
+    errors.add(:events, :invalid) if events.any? { |e| !EVENTS.include?(e) }
+  end
+
+  def strip_events
+    self.events = events.map { |str| str.strip.presence }.compact if events.present?
+  end
+
+  def generate_secret
+    self.secret = SecureRandom.hex(20) if secret.blank?
+  end
+end

app/policies/webhook_policy.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class WebhookPolicy < ApplicationPolicy
+  def index?
+    admin?
+  end
+
+  def create?
+    admin?
+  end
+
+  def show?
+    admin?
+  end
+
+  def update?
+    admin?
+  end
+
+  def enable?
+    admin?
+  end
+
+  def disable?
+    admin?
+  end
+
+  def rotate_secret?
+    admin?
+  end
+
+  def destroy?
+    admin?
+  end
+end

app/presenters/webhooks/event_presenter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Webhooks::EventPresenter < ActiveModelSerializers::Model
+  attributes :type, :created_at, :object
+
+  def initialize(type, object)
+    super()
+
+    @type       = type
+    @created_at = Time.now.utc
+    @object     = object
+  end
+end

app/serializers/rest/admin/report_serializer.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 class REST::Admin::ReportSerializer < ActiveModel::Serializer
-  attributes :id, :action_taken, :category, :comment, :created_at, :updated_at
+  attributes :id, :action_taken, :action_taken_at, :category, :comment,
+             :created_at, :updated_at
 
   has_one :account, serializer: REST::Admin::AccountSerializer
   has_one :target_account, serializer: REST::Admin::AccountSerializer

app/serializers/rest/admin/webhook_event_serializer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class REST::Admin::WebhookEventSerializer < ActiveModel::Serializer
+  def self.serializer_for(model, options)
+    case model.class.name
+    when 'Account'
+      REST::Admin::AccountSerializer
+    when 'Report'
+      REST::Admin::ReportSerializer
+    else
+      super
+    end
+  end
+
+  attributes :event, :created_at
+
+  has_one :virtual_object, key: :object
+
+  def virtual_object
+    object.object
+  end
+
+  def event
+    object.type
+  end
+end

app/services/base_service.rb

@@ -5,4 +5,8 @@ class BaseService
   include ActionView::Helpers::SanitizeHelper
 
   include RoutingHelper
+
+  def call(*)
+    raise NotImplementedError
+  end
 end

app/services/webhook_service.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class WebhookService < BaseService
+  def call(event, object)
+    @event  = Webhooks::EventPresenter.new(event, object)
+    @body   = serialize_event
+
+    webhooks_for_event.each do |webhook_id|
+      Webhooks::DeliveryWorker.perform_async(webhook_id, @body)
+    end
+  end
+
+  private
+
+  def webhooks_for_event
+    Webhook.enabled.where('? = ANY(events)', @event.type).pluck(:id)
+  end
+
+  def serialize_event
+    Oj.dump(ActiveModelSerializers::SerializableResource.new(@event, serializer: REST::Admin::WebhookEventSerializer, scope: nil, scope_name: :current_user).as_json)
+  end
+end

app/validators/url_validator.rb

@@ -2,7 +2,7 @@
 
 class URLValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
-    record.errors.add(attribute, I18n.t('applications.invalid_url')) unless compliant?(value)
+    record.errors.add(attribute, :invalid) unless compliant?(value)
   end
 
   private

app/views/admin/webhooks/_form.html.haml

@@ -0,0 +1,11 @@
+= simple_form_for @webhook, url: @webhook.new_record? ? admin_webhooks_path : admin_webhook_path(@webhook) do |f|
+  = render 'shared/error_messages', object: @webhook
+
+  .fields-group
+    = f.input :url, wrapper: :with_block_label, input_html: { placeholder: 'https://' }
+
+  .fields-group
+    = f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
+
+  .actions
+    = f.button :button, @webhook.new_record? ? t('admin.webhooks.add_new') : t('generic.save_changes'), type: :submit

app/views/admin/webhooks/_webhook.html.haml

@@ -0,0 +1,19 @@
+.applications-list__item
+  = link_to admin_webhook_path(webhook), class: 'announcements-list__item__title' do
+    = fa_icon 'inbox'
+    = webhook.url
+
+  .announcements-list__item__action-bar
+    .announcements-list__item__meta
+      - if webhook.enabled?
+        %span.positive-hint= t('admin.webhooks.enabled')
+      - else
+        %span.negative-hint= t('admin.webhooks.disabled')
+
+      •
+
+      %abbr{ title: webhook.events.join(', ') }= t('admin.webhooks.enabled_events', count: webhook.events.size)
+
+    %div
+      = table_link_to 'pencil', t('admin.webhooks.edit'), edit_admin_webhook_path(webhook) if can?(:update, webhook)
+      = table_link_to 'trash', t('admin.webhooks.delete'), admin_webhook_path(webhook), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') } if can?(:destroy, webhook)

app/views/admin/webhooks/edit.html.haml

@@ -0,0 +1,4 @@
+- content_for :page_title do
+  = t('admin.webhooks.edit')
+
+= render partial: 'form'

app/views/admin/webhooks/index.html.haml

@@ -0,0 +1,18 @@
+- content_for :page_title do
+  = t('admin.webhooks.title')
+
+- content_for :heading_actions do
+  = link_to t('admin.webhooks.add_new'), new_admin_webhook_path, class: 'button' if can?(:create, :webhook)
+
+%p= t('admin.webhooks.description_html')
+
+%hr.spacer/
+
+- if @webhooks.empty?
+  %div.muted-hint.center-text
+    = t 'admin.webhooks.empty'
+- else
+  .applications-list
+    = render partial: 'webhook', collection: @webhooks
+
+  = paginate @webhooks

app/views/admin/webhooks/new.html.haml

@@ -0,0 +1,4 @@
+- content_for :page_title do
+  = t('admin.webhooks.new')
+
+= render partial: 'form'

app/views/admin/webhooks/show.html.haml

@@ -0,0 +1,34 @@
+- content_for :page_title do
+  = t('admin.webhooks.title')
+
+- content_for :heading do
+  %h2
+    %small
+      = fa_icon 'inbox'
+      = t('admin.webhooks.webhook')
+    = @webhook.url
+
+- content_for :heading_actions do
+  = link_to t('admin.webhooks.edit'), edit_admin_webhook_path, class: 'button' if can?(:update, @webhook)
+
+.table-wrapper
+  %table.table.horizontal-table
+    %tbody
+      %tr
+        %th= t('admin.webhooks.status')
+        %td
+          - if @webhook.enabled?
+            %span.positive-hint= t('admin.webhooks.enabled')
+            = table_link_to 'power-off', t('admin.webhooks.disable'), disable_admin_webhook_path(@webhook), method: :post if can?(:disable, @webhook)
+          - else
+            %span.negative-hint= t('admin.webhooks.disabled')
+            = table_link_to 'power-off', t('admin.webhooks.enable'), enable_admin_webhook_path(@webhook), method: :post if can?(:enable, @webhook)
+      %tr
+        %th= t('admin.webhooks.events')
+        %td
+          %abbr{ title: @webhook.events.join(', ') }= t('admin.webhooks.enabled_events', count: @webhook.events.size)
+      %tr
+        %th= t('admin.webhooks.secret')
+        %td
+          %samp= @webhook.secret
+          = table_link_to 'refresh', t('admin.webhooks.rotate_secret'), rotate_admin_webhook_secret_path(@webhook), method: :post if can?(:rotate_secret, @webhook)

app/views/layouts/admin.html.haml

@@ -23,6 +23,9 @@
     .content-wrapper
       .content
         .content-heading
+          - if content_for?(:heading)
+            = yield :heading
+          - else
             %h2= yield :page_title
 
           - if :heading_actions

app/workers/trigger_webhook_worker.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class TriggerWebhookWorker
+  include Sidekiq::Worker
+
+  def perform(event, class_name, id)
+    object = class_name.constantize.find(id)
+    WebhookService.new.call(event, object)
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

app/workers/webhooks/delivery_worker.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class Webhooks::DeliveryWorker
+  include Sidekiq::Worker
+  include JsonLdHelper
+
+  sidekiq_options queue: 'push', retry: 16, dead: false
+
+  def perform(webhook_id, body)
+    @webhook   = Webhook.find(webhook_id)
+    @body      = body
+    @response  = nil
+
+    perform_request
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+
+  private
+
+  def perform_request
+    request = Request.new(:post, @webhook.url, body: @body)
+
+    request.add_headers(
+      'Content-Type' => 'application/json',
+      'X-Hub-Signature' => "sha256=#{signature}"
+    )
+
+    request.perform do |response|
+      raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
+    end
+  end
+
+  def signature
+    OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'), @webhook.secret, @body)
+  end
+end

config/locales/activerecord.en.yml

@@ -21,6 +21,14 @@ en:
             username:
               invalid: must contain only letters, numbers and underscores
               reserved: is reserved
+        admin/webhook:
+          attributes:
+            url:
+              invalid: is not a valid URL
+        doorkeeper/application:
+          attributes:
+            website:
+              invalid: is not a valid URL
         status:
           attributes:
             reblog:

config/locales/en.yml

@@ -852,6 +852,26 @@ en:
       edit_preset: Edit warning preset
       empty: You haven't defined any warning presets yet.
       title: Manage warning presets
+    webhooks:
+      add_new: Add endpoint
+      delete: Delete
+      description_html: A <strong>webhook</strong> enables Mastodon to push <strong>real-time notifications</strong> about chosen events to your own application, so your application can <strong>automatically trigger reactions</strong>.
+      disable: Disable
+      disabled: Disabled
+      edit: Edit endpoint
+      empty: You don't have any webhook endpoints configured yet.
+      enable: Enable
+      enabled: Active
+      enabled_events:
+        one: 1 enabled event
+        other: "%{count} enabled events"
+      events: Events
+      new: New webhook
+      rotate_secret: Rotate secret
+      secret: Signing secret
+      status: Status
+      title: Webhooks
+      webhook: Webhook
   admin_mailer:
     new_appeal:
       actions:
@@ -916,7 +936,6 @@ en:
   applications:
     created: Application successfully created
     destroyed: Application successfully deleted
-    invalid_url: The provided URL is invalid
     regenerate_token: Regenerate access token
     token_regenerated: Access token successfully regenerated
     warning: Be very careful with this data. Never share it with anyone!

config/locales/simple_form.en.yml

@@ -91,6 +91,9 @@ en:
         name: You can only change the casing of the letters, for example, to make it more readable
       user:
         chosen_languages: When checked, only posts in selected languages will be displayed in public timelines
+      webhook:
+        events: Select events to send
+        url: Where events will be sent to
     labels:
       account:
         fields:
@@ -219,6 +222,9 @@ en:
         name: Hashtag
         trendable: Allow this hashtag to appear under trends
         usable: Allow posts to use this hashtag
+      webhook:
+        events: Enabled events
+        url: Endpoint URL
     'no': 'No'
     recommended: Recommended
     required:

config/navigation.rb

@@ -56,6 +56,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :rules, safe_join([fa_icon('gavel fw'), t('admin.rules.title')]), admin_rules_path, highlights_on: %r{/admin/rules}
       s.item :announcements, safe_join([fa_icon('bullhorn fw'), t('admin.announcements.title')]), admin_announcements_path, highlights_on: %r{/admin/announcements}
       s.item :custom_emojis, safe_join([fa_icon('smile-o fw'), t('admin.custom_emojis.title')]), admin_custom_emojis_url, highlights_on: %r{/admin/custom_emojis}
+      s.item :webhooks, safe_join([fa_icon('inbox fw'), t('admin.webhooks.title')]), admin_webhooks_path, highlights_on: %r{/admin/webhooks}
       s.item :relays, safe_join([fa_icon('exchange fw'), t('admin.relays.title')]), admin_relays_url, if: -> { current_user.admin? && !whitelist_mode? }, highlights_on: %r{/admin/relays}
       s.item :sidekiq, safe_join([fa_icon('diamond fw'), 'Sidekiq']), sidekiq_url, link_html: { target: 'sidekiq' }, if: -> { current_user.admin? }
       s.item :pghero, safe_join([fa_icon('database fw'), 'PgHero']), pghero_url, link_html: { target: 'pghero' }, if: -> { current_user.admin? }

config/routes.rb

@@ -235,6 +235,17 @@ Rails.application.routes.draw do
 
     resources :rules
 
+    resources :webhooks do
+      member do
+        post :enable
+        post :disable
+      end
+
+      resource :secret, only: [], controller: 'webhooks/secrets' do
+        post :rotate
+      end
+    end
+
     resources :reports, only: [:index, :show] do
       resources :actions, only: [:create], controller: 'reports/actions'
 

db/migrate/20220606044941_create_webhooks.rb

@@ -0,0 +1,12 @@
+class CreateWebhooks < ActiveRecord::Migration[6.1]
+  def change
+    create_table :webhooks do |t|
+      t.string :url, null: false, index: { unique: true }
+      t.string :events, array: true, null: false, default: []
+      t.string :secret, null: false, default: ''
+      t.boolean :enabled, null: false, default: true
+
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2022_05_27_114923) do
+ActiveRecord::Schema.define(version: 2022_06_06_044941) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1035,6 +1035,16 @@ ActiveRecord::Schema.define(version: 2022_05_27_114923) do
     t.index ["user_id"], name: "index_webauthn_credentials_on_user_id"
   end
 
+  create_table "webhooks", force: :cascade do |t|
+    t.string "url", null: false
+    t.string "events", default: [], null: false, array: true
+    t.string "secret", default: "", null: false
+    t.boolean "enabled", default: true, null: false
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["url"], name: "index_webhooks_on_url", unique: true
+  end
+
   add_foreign_key "account_aliases", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "conversations", on_delete: :cascade

spec/fabricators/webhook_fabricator.rb

@@ -0,0 +1,5 @@
+Fabricator(:webhook) do
+  url { Faker::Internet.url }
+  secret { SecureRandom.hex }
+  events { Webhook::EVENTS }
+end

spec/models/webhook_spec.rb

@@ -0,0 +1,32 @@
+require 'rails_helper'
+
+RSpec.describe Webhook, type: :model do
+  let(:webhook) { Fabricate(:webhook) }
+
+  describe '#rotate_secret!' do
+    it 'changes the secret' do
+      previous_value = webhook.secret
+      webhook.rotate_secret!
+      expect(webhook.secret).to_not be_blank
+      expect(webhook.secret).to_not eq previous_value
+    end
+  end
+
+  describe '#enable!' do
+    before do
+      webhook.disable!
+    end
+
+    it 'enables the webhook' do
+      webhook.enable!
+      expect(webhook.enabled?).to be true
+    end
+  end
+
+  describe '#disable!' do
+    it 'disables the webhook' do
+      webhook.disable!
+      expect(webhook.enabled?).to be false
+    end
+  end
+end

spec/validators/url_validator_spec.rb

@@ -19,7 +19,7 @@ RSpec.describe URLValidator, type: :validator do
       let(:compliant) { false }
 
       it 'calls errors.add' do
-        expect(errors).to have_received(:add).with(attribute, I18n.t('applications.invalid_url'))
+        expect(errors).to have_received(:add).with(attribute, :invalid)
       end
     end