Merge branch 'develop' into fix/ldap-auth-issues

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- **Breaking:** Added the ObjectAgePolicy to the default set of MRFs. This will delist and strip the follower collection of any message received that is older than 7 days. This will stop users from seeing very old messages in the timelines. The messages can still be viewed on the user's page and in conversations. They also still trigger notifications.
 - **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
 - **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated.
 - In Conversations, return only direct messages as `last_status`
@@ -103,6 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix CSP policy generation to include remote Captcha services
 - Fix edge case where MediaProxy truncates media, usually caused when Caddy is serving content for the other Federated instance.
 - Emoji Packs could not be listed when instance was set to `public: false`
+- Fix whole_word always returning false on filter get requests
 
 ## [Unreleased (patch)]
 

config/config.exs

@@ -515,7 +515,13 @@ config :pleroma, Pleroma.User,
     "user-search",
     "user_exists",
     "users",
-    "web"
+    "web",
+    "verify_credentials",
+    "update_credentials",
+    "relationships",
+    "search",
+    "confirmation_resend",
+    "mfa"
   ],
   email_blacklist: []
 

docs/configuration/cheatsheet.md

@@ -858,9 +858,6 @@ Warning: it's discouraged to use this feature because of the associated security
 
 ### :auth
 
-* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator.
-* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication.
-
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.

lib/pleroma/application.ex

@@ -47,6 +47,7 @@ defmodule Pleroma.Application do
     Pleroma.ApplicationRequirements.verify!()
     setup_instrumenters()
     load_custom_modules()
+    check_system_commands()
     Pleroma.Docs.JSON.compile()
 
     adapter = Application.get_env(:tesla, :adapter)
@@ -249,4 +250,21 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(_, _), do: []
+
+  defp check_system_commands do
+    filters = Config.get([Pleroma.Upload, :filters])
+
+    check_filter = fn filter, command_required ->
+      with true <- filter in filters,
+           false <- Pleroma.Utils.command_available?(command_required) do
+        Logger.error(
+          "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found"
+        )
+      end
+    end
+
+    check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool")
+    check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify")
+    check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+  end
 end

lib/pleroma/config.ex

@@ -11,12 +11,10 @@ defmodule Pleroma.Config do
 
   def get([key], default), do: get(key, default)
 
-  def get([parent_key | keys], default) do
+  def get([_ | _] = path, default) do
-    case :pleroma
+    case fetch(path) do
-         |> Application.get_env(parent_key)
+      {:ok, value} -> value
-         |> get_in(keys) do
+      :error -> default
-      nil -> default
-      any -> any
     end
   end
 
@@ -34,6 +32,24 @@ defmodule Pleroma.Config do
     end
   end
 
+  def fetch(key) when is_atom(key), do: fetch([key])
+
+  def fetch([root_key | keys]) do
+    Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn
+      key, {:ok, config} when is_map(config) or is_list(config) ->
+        case Access.fetch(config, key) do
+          :error ->
+            {:halt, :error}
+
+          value ->
+            {:cont, value}
+        end
+
+      _key, _config ->
+        {:halt, :error}
+    end)
+  end
+
   def put([key], value), do: put(key, value)
 
   def put([parent_key | keys], value) do
@@ -50,13 +66,16 @@ defmodule Pleroma.Config do
 
   def delete([key]), do: delete(key)
 
-  def delete([parent_key | keys]) do
+  def delete([parent_key | keys] = path) do
+    with {:ok, _} <- fetch(path) do
       {_, parent} =
-      Application.get_env(:pleroma, parent_key)
+        parent_key
+        |> get()
         |> get_and_update_in(keys, fn _ -> :pop end)
 
       Application.put_env(:pleroma, parent_key, parent)
     end
+  end
 
   def delete(key) do
     Application.delete_env(:pleroma, key)

lib/pleroma/object.ex

@@ -255,6 +255,10 @@ defmodule Pleroma.Object do
     end
   end
 
+  defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true
+
+  defp poll_is_multiple?(_), do: false
+
   def decrease_replies_count(ap_id) do
     Object
     |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
@@ -281,10 +285,10 @@ defmodule Pleroma.Object do
   def increase_vote_count(ap_id, name, actor) do
     with %Object{} = object <- Object.normalize(ap_id),
          "Question" <- object.data["type"] do
-      multiple = Map.has_key?(object.data, "anyOf")
+      key = if poll_is_multiple?(object), do: "anyOf", else: "oneOf"
 
       options =
-        (object.data["anyOf"] || object.data["oneOf"] || [])
+        object.data[key]
         |> Enum.map(fn
           %{"name" => ^name} = option ->
             Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1))
@@ -296,11 +300,8 @@ defmodule Pleroma.Object do
       voters = [actor | object.data["voters"] || []] |> Enum.uniq()
 
       data =
-        if multiple do
+        object.data
-          Map.put(object.data, "anyOf", options)
+        |> Map.put(key, options)
-        else
-          Map.put(object.data, "oneOf", options)
-        end
         |> Map.put("voters", voters)
 
       object

lib/pleroma/object/containment.ex

@@ -55,7 +55,7 @@ defmodule Pleroma.Object.Containment do
   defp compare_uris(_id_uri, _other_uri), do: :error
 
   @doc """
-  Checks that an imported AP object's actor matches the domain it came from.
+  Checks that an imported AP object's actor matches the host it came from.
   """
   def contain_origin(_id, %{"actor" => nil}), do: :error
 

lib/pleroma/object/fetcher.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Repo
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
+  alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
 
@@ -23,21 +24,39 @@ defmodule Pleroma.Object.Fetcher do
     Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
   end
 
-  defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do
+  defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
     internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
 
-    Map.merge(data, internal_fields)
+    Map.merge(new_data, internal_fields)
   end
 
-  defp maybe_reinject_internal_fields(data, _), do: data
+  defp maybe_reinject_internal_fields(_, new_data), do: new_data
 
   @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
-  defp reinject_object(struct, data) do
+  defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do
-    Logger.debug("Reinjecting object #{data["id"]}")
+    Logger.debug("Reinjecting object #{new_data["id"]}")
 
-    with data <- Transmogrifier.fix_object(data),
+    with new_data <- Transmogrifier.fix_object(new_data),
-         data <- maybe_reinject_internal_fields(data, struct),
+         data <- maybe_reinject_internal_fields(object, new_data),
-         changeset <- Object.change(struct, %{data: data}),
+         {:ok, data, _} <- ObjectValidator.validate(data, %{}),
+         changeset <- Object.change(object, %{data: data}),
+         changeset <- touch_changeset(changeset),
+         {:ok, object} <- Repo.insert_or_update(changeset),
+         {:ok, object} <- Object.set_cache(object) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
+  defp reinject_object(%Object{} = object, new_data) do
+    Logger.debug("Reinjecting object #{new_data["id"]}")
+
+    with new_data <- Transmogrifier.fix_object(new_data),
+         data <- maybe_reinject_internal_fields(object, new_data),
+         changeset <- Object.change(object, %{data: data}),
          changeset <- touch_changeset(changeset),
          {:ok, object} <- Repo.insert_or_update(changeset),
          {:ok, object} <- Object.set_cache(object) do
@@ -51,8 +70,8 @@ defmodule Pleroma.Object.Fetcher do
 
   def refetch_object(%Object{data: %{"id" => id}} = object) do
     with {:local, false} <- {:local, Object.local?(object)},
-         {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+         {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id),
-         {:ok, object} <- reinject_object(object, data) do
+         {:ok, object} <- reinject_object(object, new_data) do
       {:ok, object}
     else
       {:local, true} -> {:ok, object}

lib/pleroma/upload/filter/exiftool.ex

@@ -9,9 +9,17 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   """
   @behaviour Pleroma.Upload.Filter
 
+  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true)
+    try do
-    :ok
+      case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
+        {_response, 0} -> :ok
+        {error, 1} -> {:error, error}
+      end
+    rescue
+      _e in ErlangError ->
+        {:error, "exiftool command not found"}
+    end
   end
 
   def filter(_), do: :ok

lib/pleroma/upload/filter/mogrifun.ex

@@ -34,10 +34,15 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
     [{"fill", "yellow"}, {"tint", "40"}]
   ]
 
+  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
+    try do
       Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
-
       :ok
+    rescue
+      _e in ErlangError ->
+        {:error, "mogrify command not found"}
+    end
   end
 
   def filter(_), do: :ok

lib/pleroma/upload/filter/mogrify.ex

@@ -8,11 +8,15 @@ defmodule Pleroma.Upload.Filter.Mogrify do
   @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
   @type conversions :: conversion() | [conversion()]
 
+  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    filters = Pleroma.Config.get!([__MODULE__, :args])
+    try do
-
+      do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
-    do_filter(file, filters)
       :ok
+    rescue
+      _e in ErlangError ->
+        {:error, "mogrify command not found"}
+    end
   end
 
   def filter(_), do: :ok

lib/pleroma/utils.ex

@@ -9,4 +9,19 @@ defmodule Pleroma.Utils do
     |> Enum.map(&Path.join(dir, &1))
     |> Kernel.ParallelCompiler.compile()
   end
+
+  @doc """
+  POSIX-compliant check if command is available in the system
+
+  ## Examples
+      iex> command_available?("git")
+      true
+      iex> command_available?("wrongcmd")
+      false
+
+  """
+  @spec command_available?(String.t()) :: boolean()
+  def command_available?(command) do
+    match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -66,7 +66,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp check_remote_limit(_), do: true
 
-  defp increase_note_count_if_public(actor, object) do
+  def increase_note_count_if_public(actor, object) do
     if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor}
   end
 
@@ -85,17 +85,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  defp increase_poll_votes_if_vote(%{
+  @object_types ["ChatMessage", "Question", "Answer"]
-         "object" => %{"inReplyTo" => reply_ap_id, "name" => name},
-         "type" => "Create",
-         "actor" => actor
-       }) do
-    Object.increase_vote_count(reply_ap_id, name, actor)
-  end
-
-  defp increase_poll_votes_if_vote(_create_data), do: :noop
-
-  @object_types ["ChatMessage"]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
@@ -258,7 +248,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     with {:ok, activity} <- insert(create_data, local, fake),
          {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
-         _ <- increase_poll_votes_if_vote(create_data),
          {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          _ <- notify_and_stream(activity),

lib/pleroma/web/activity_pub/builder.ex

@@ -80,6 +80,13 @@ defmodule Pleroma.Web.ActivityPub.Builder do
   end
 
   def create(actor, object, recipients) do
+    context =
+      if is_map(object) do
+        object["context"]
+      else
+        nil
+      end
+
     {:ok,
      %{
        "id" => Utils.generate_activity_id(),
@@ -88,7 +95,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do
        "object" => object,
        "type" => "Create",
        "published" => DateTime.utc_now() |> DateTime.to_iso8601()
-     }, []}
+     }
+     |> Pleroma.Maps.put_if_present("context", context), []}
   end
 
   def chat_message(actor, recipient, content, opts \\ []) do
@@ -115,6 +123,22 @@ defmodule Pleroma.Web.ActivityPub.Builder do
     end
   end
 
+  def answer(user, object, name) do
+    {:ok,
+     %{
+       "type" => "Answer",
+       "actor" => user.ap_id,
+       "attributedTo" => user.ap_id,
+       "cc" => [object.data["actor"]],
+       "to" => [],
+       "name" => name,
+       "inReplyTo" => object.data["id"],
+       "context" => object.data["context"],
+       "published" => DateTime.utc_now() |> DateTime.to_iso8601(),
+       "id" => Utils.generate_object_id()
+     }, []}
+  end
+
   @spec tombstone(String.t(), String.t()) :: {:ok, map(), keyword()}
   def tombstone(actor, id) do
     {:ok,

lib/pleroma/web/activity_pub/object_validator.ex

@@ -14,13 +14,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator
 
@@ -112,17 +115,40 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => "Question"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> QuestionValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Answer"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> AnswerValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => "EmojiReact"} = object, meta) do
     with {:ok, object} <-
            object
            |> EmojiReactValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object |> Map.from_struct())
+      object = stringify_keys(object)
       {:ok, object, meta}
     end
   end
 
-  def validate(%{"type" => "Create", "object" => object} = create_activity, meta) do
+  def validate(
+        %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity,
+        meta
+      ) do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -134,12 +160,28 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(
+        %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
+        meta
+      )
+      when objtype in ["Question", "Answer"] do
+    with {:ok, object_data} <- cast_and_apply(object),
+         meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
+         {:ok, create_activity} <-
+           create_activity
+           |> CreateGenericValidator.cast_and_validate(meta)
+           |> Ecto.Changeset.apply_action(:insert) do
+      create_activity = stringify_keys(create_activity)
+      {:ok, create_activity, meta}
+    end
+  end
+
   def validate(%{"type" => "Announce"} = object, meta) do
     with {:ok, object} <-
            object
            |> AnnounceValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object |> Map.from_struct())
+      object = stringify_keys(object)
       {:ok, object, meta}
     end
   end
@@ -148,8 +190,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     ChatMessageValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Question"} = object) do
+    QuestionValidator.cast_and_apply(object)
+  end
+
+  def cast_and_apply(%{"type" => "Answer"} = object) do
+    AnswerValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
+  # is_struct/1 isn't present in Elixir 1.8.x
   def stringify_keys(%{__struct__: _} = object) do
     object
     |> Map.from_struct()

lib/pleroma/web/activity_pub/object_validators/answer_validator.ex

@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, {:array, :string}, default: [])
+    field(:cc, {:array, :string}, default: [])
+
+    # is this actually needed?
+    field(:bto, {:array, :string}, default: [])
+    field(:bcc, {:array, :string}, default: [])
+
+    field(:type, :string)
+    field(:name, :string)
+    field(:inReplyTo, :string)
+    field(:attributedTo, ObjectValidators.ObjectID)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data()
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Answer"])
+    |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end

lib/pleroma/web/activity_pub/object_validators/common_validations.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
   alias Pleroma.Object
   alias Pleroma.User
 
-  def validate_recipients_presence(cng, fields \\ [:to, :cc]) do
+  def validate_any_presence(cng, fields) do
     non_empty =
       fields
       |> Enum.map(fn field -> get_field(cng, field) end)
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
       fields
       |> Enum.reduce(cng, fn field, cng ->
         cng
-        |> add_error(field, "no recipients in any field")
+        |> add_error(field, "none of #{inspect(fields)} present")
       end)
     end
   end
@@ -82,4 +82,60 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
 
     if actor_cng.valid?, do: actor_cng, else: object_cng
   end
+
+  def validate_host_match(cng, fields \\ [:id, :actor]) do
+    if same_domain?(cng, fields) do
+      cng
+    else
+      fields
+      |> Enum.reduce(cng, fn field, cng ->
+        cng
+        |> add_error(field, "hosts of #{inspect(fields)} aren't matching")
+      end)
+    end
+  end
+
+  def validate_fields_match(cng, fields) do
+    if map_unique?(cng, fields) do
+      cng
+    else
+      fields
+      |> Enum.reduce(cng, fn field, cng ->
+        cng
+        |> add_error(field, "Fields #{inspect(fields)} aren't matching")
+      end)
+    end
+  end
+
+  defp map_unique?(cng, fields, func \\ & &1) do
+    Enum.reduce_while(fields, nil, fn field, acc ->
+      value =
+        cng
+        |> get_field(field)
+        |> func.()
+
+      case {value, acc} do
+        {value, nil} -> {:cont, value}
+        {value, value} -> {:cont, value}
+        _ -> {:halt, false}
+      end
+    end)
+  end
+
+  def same_domain?(cng, fields \\ [:actor, :object]) do
+    map_unique?(cng, fields, fn value -> URI.parse(value).host end)
+  end
+
+  # This figures out if a user is able to create, delete or modify something
+  # based on the domain and superuser status
+  def validate_modification_rights(cng) do
+    actor = User.get_cached_by_ap_id(get_field(cng, :actor))
+
+    if User.superuser?(actor) || same_domain?(cng) do
+      cng
+    else
+      cng
+      |> add_error(:actor, "is not allowed to modify object")
+    end
+  end
 end

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -0,0 +1,133 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Code based on CreateChatMessageValidator
+# NOTES
+# - doesn't embed, will only get the object id
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Object
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:type, :string)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:object, ObjectValidators.ObjectID)
+    field(:expires_at, ObjectValidators.DateTime)
+
+    # Should be moved to object, done for CommonAPI.Utils.make_context
+    field(:context, :string)
+  end
+
+  def cast_data(data, meta \\ []) do
+    data = fix(data, meta)
+
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data, meta \\ []) do
+    data
+    |> cast_data(meta)
+    |> validate_data(meta)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+  end
+
+  defp fix_context(data, meta) do
+    if object = meta[:object_data] do
+      Map.put_new(data, "context", object["context"])
+    else
+      data
+    end
+  end
+
+  defp fix(data, meta) do
+    data
+    |> fix_context(meta)
+  end
+
+  def validate_data(cng, meta \\ []) do
+    cng
+    |> validate_required([:actor, :type, :object])
+    |> validate_inclusion(:type, ["Create"])
+    |> validate_actor_presence()
+    |> validate_any_presence([:to, :cc])
+    |> validate_actors_match(meta)
+    |> validate_context_match(meta)
+    |> validate_object_nonexistence()
+    |> validate_object_containment()
+  end
+
+  def validate_object_containment(cng) do
+    actor = get_field(cng, :actor)
+
+    cng
+    |> validate_change(:object, fn :object, object_id ->
+      %URI{host: object_id_host} = URI.parse(object_id)
+      %URI{host: actor_host} = URI.parse(actor)
+
+      if object_id_host == actor_host do
+        []
+      else
+        [{:object, "The host of the object id doesn't match with the host of the actor"}]
+      end
+    end)
+  end
+
+  def validate_object_nonexistence(cng) do
+    cng
+    |> validate_change(:object, fn :object, object_id ->
+      if Object.get_cached_by_ap_id(object_id) do
+        [{:object, "The object to create already exists"}]
+      else
+        []
+      end
+    end)
+  end
+
+  def validate_actors_match(cng, meta) do
+    attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"]
+
+    cng
+    |> validate_change(:actor, fn :actor, actor ->
+      if actor == attributed_to do
+        []
+      else
+        [{:actor, "Actor doesn't match with object attributedTo"}]
+      end
+    end)
+  end
+
+  def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do
+    cng
+    |> validate_change(:context, fn :context, context ->
+      if context == object_context do
+        []
+      else
+        [{:context, "context field not matching between Create and object (#{object_context})"}]
+      end
+    end)
+  end
+
+  def validate_context_match(cng, _), do: cng
+end

lib/pleroma/web/activity_pub/object_validators/delete_validator.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
 
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.User
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -59,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Delete"])
     |> validate_actor_presence()
-    |> validate_deletion_rights()
+    |> validate_modification_rights()
     |> validate_object_or_user_presence(allowed_types: @deletable_types)
     |> add_deleted_activity_id()
   end
@@ -68,31 +67,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     !same_domain?(cng)
   end
 
-  defp same_domain?(cng) do
-    actor_uri =
-      cng
-      |> get_field(:actor)
-      |> URI.parse()
-
-    object_uri =
-      cng
-      |> get_field(:object)
-      |> URI.parse()
-
-    object_uri.host == actor_uri.host
-  end
-
-  def validate_deletion_rights(cng) do
-    actor = User.get_cached_by_ap_id(get_field(cng, :actor))
-
-    if User.superuser?(actor) || same_domain?(cng) do
-      cng
-    else
-      cng
-      |> add_error(:actor, "is not allowed to delete object")
-    end
-  end
-
   def cast_and_validate(data) do
     data
     |> cast_data

lib/pleroma/web/activity_pub/object_validators/note_validator.ex

@@ -34,7 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:replies_count, :integer, default: 0)
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
-    field(:inRepyTo, :string)
+    field(:inReplyTo, :string)
     field(:uri, ObjectValidators.Uri)
 
     field(:likes, {:array, :string}, default: [])

lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex

@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  @primary_key false
+
+  embedded_schema do
+    field(:name, :string)
+
+    embeds_one :replies, Replies, primary_key: false do
+      field(:totalItems, :integer)
+      field(:type, :string)
+    end
+
+    field(:type, :string)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, [:name, :type])
+    |> cast_embed(:replies, with: &replies_changeset/2)
+    |> validate_inclusion(:type, ["Note"])
+    |> validate_required([:name, :type])
+  end
+
+  def replies_changeset(struct, data) do
+    struct
+    |> cast(data, [:totalItems, :type])
+    |> validate_inclusion(:type, ["Collection"])
+    |> validate_required([:type])
+  end
+end

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -0,0 +1,127 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
+  alias Pleroma.Web.ActivityPub.Utils
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  # Extends from NoteValidator
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, {:array, :string}, default: [])
+    field(:cc, {:array, :string}, default: [])
+    field(:bto, {:array, :string}, default: [])
+    field(:bcc, {:array, :string}, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+    field(:content, :string)
+    field(:context, :string)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:summary, :string)
+    field(:published, ObjectValidators.DateTime)
+    # TODO: Write type
+    field(:emoji, :map, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, :string)
+    field(:uri, ObjectValidators.Uri)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    field(:likes, {:array, :string}, default: [])
+    field(:announcements, {:array, :string}, default: [])
+
+    field(:closed, ObjectValidators.DateTime)
+    field(:voters, {:array, ObjectValidators.ObjectID}, default: [])
+    embeds_many(:anyOf, QuestionOptionsValidator)
+    embeds_many(:oneOf, QuestionOptionsValidator)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix_closed(data) do
+    cond do
+      is_binary(data["closed"]) -> data
+      is_binary(data["endTime"]) -> Map.put(data, "closed", data["endTime"])
+      true -> Map.drop(data, ["closed"])
+    end
+  end
+
+  # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
+  defp fix_defaults(data) do
+    %{data: %{"id" => context}, id: context_id} =
+      Utils.create_context(data["context"] || data["conversation"])
+
+    data
+    |> Map.put_new_lazy("published", &Utils.make_date/0)
+    |> Map.put_new("context", context)
+    |> Map.put_new("context_id", context_id)
+  end
+
+  defp fix_attribution(data) do
+    data
+    |> Map.put_new("actor", data["attributedTo"])
+  end
+
+  defp fix(data) do
+    data
+    |> fix_attribution()
+    |> fix_closed()
+    |> fix_defaults()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment])
+    |> cast_embed(:attachment)
+    |> cast_embed(:anyOf)
+    |> cast_embed(:oneOf)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Question"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_any_presence([:oneOf, :anyOf])
+    |> CommonValidations.validate_host_match()
+  end
+end

lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
   embedded_schema do
     field(:type, :string)
     field(:href, ObjectValidators.Uri)
-    field(:mediaType, :string)
+    field(:mediaType, :string, default: "application/octet-stream")
   end
 
   def changeset(struct, data) do

lib/pleroma/web/activity_pub/side_effects.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   """
   alias Pleroma.Activity
   alias Pleroma.Activity.Ir.Topics
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
   alias Pleroma.FollowingRelationship
@@ -19,6 +20,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
+  alias Pleroma.Workers.BackgroundWorker
 
   def handle(object, meta \\ [])
 
@@ -135,10 +137,26 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # Tasks this handles
   # - Actually create object
   # - Rollback if we couldn't create it
+  # - Increase the user note count
+  # - Increase the reply count
+  # - Increase replies count
+  # - Set up ActivityExpiration
   # - Set up notifications
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
-    with {:ok, _object, meta} <- handle_object_creation(meta[:object_data], meta) do
+    with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta),
+         %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
+      {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
+
+      if in_reply_to = object.data["inReplyTo"] do
+        Object.increase_replies_count(in_reply_to)
+      end
+
+      if expires_at = activity.data["expires_at"] do
+        ActivityExpiration.create(activity, expires_at)
+      end
+
+      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
 
       meta =
         meta
@@ -268,9 +286,27 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     end
   end
 
+  def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do
+    with {:ok, object, meta} <- Pipeline.common_pipeline(object_map, meta) do
+      Object.increase_vote_count(
+        object.data["inReplyTo"],
+        object.data["name"],
+        object.data["actor"]
+      )
+
+      {:ok, object, meta}
+    end
+  end
+
+  def handle_object_creation(%{"type" => "Question"} = object, meta) do
+    with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
+      {:ok, object, meta}
+    end
+  end
+
   # Nothing to do
-  def handle_object_creation(object) do
+  def handle_object_creation(object, meta) do
-    {:ok, object}
+    {:ok, object, meta}
   end
 
   defp undo_like(nil, object), do: delete_object(object)

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -157,7 +157,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
-    Map.put(object, "actor", Containment.get_actor(%{"actor" => actor}))
+    actor = Containment.get_actor(%{"actor" => actor})
+
+    # TODO: Remove actor field for Objects
+    object
+    |> Map.put("actor", actor)
+    |> Map.put("attributedTo", actor)
   end
 
   def fix_in_reply_to(object, options \\ [])
@@ -240,13 +245,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
         if href do
           attachment_url =
-            %{"href" => href}
+            %{
+              "href" => href,
+              "type" => Map.get(url || %{}, "type", "Link")
+            }
             |> Maps.put_if_present("mediaType", media_type)
-            |> Maps.put_if_present("type", Map.get(url || %{}, "type"))
 
-          %{"url" => [attachment_url]}
+          %{
+            "url" => [attachment_url],
+            "type" => data["type"] || "Document"
+          }
           |> Maps.put_if_present("mediaType", media_type)
-          |> Maps.put_if_present("type", data["type"])
           |> Maps.put_if_present("name", data["name"])
         else
           nil
@@ -419,6 +428,29 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end)
   end
 
+  # Compatibility wrapper for Mastodon votes
+  defp handle_create(%{"object" => %{"type" => "Answer"}} = data, _user) do
+    handle_incoming(data)
+  end
+
+  defp handle_create(%{"object" => object} = data, user) do
+    %{
+      to: data["to"],
+      object: object,
+      actor: user,
+      context: object["context"],
+      local: false,
+      published: data["published"],
+      additional:
+        Map.take(data, [
+          "cc",
+          "directMessage",
+          "id"
+        ])
+    }
+    |> ActivityPub.create()
+  end
+
   def handle_incoming(data, options \\ [])
 
   # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them
@@ -457,30 +489,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do
+      when objtype in ["Article", "Event", "Note", "Video", "Page", "Audio"] do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor) do
-         data <- Map.put(data, "actor", actor) |> fix_addressing() do
+      data =
-      object = fix_object(object, options)
+        data
+        |> Map.put("object", fix_object(object, options))
+        |> Map.put("actor", actor)
+        |> fix_addressing()
 
-      params = %{
+      with {:ok, created_activity} <- handle_create(data, user) do
-        to: data["to"],
-        object: object,
-        actor: user,
-        context: object["context"],
-        local: false,
-        published: data["published"],
-        additional:
-          Map.take(data, [
-            "cc",
-            "directMessage",
-            "id"
-          ])
-      }
-
-      with {:ok, created_activity} <- ActivityPub.create(params) do
         reply_depth = (options[:depth] || 0) + 1
 
         if Federator.allowed_thread_distance?(reply_depth) do
@@ -613,6 +633,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> handle_incoming(options)
   end
 
+  def handle_incoming(
+        %{"type" => "Create", "object" => %{"type" => objtype}} = data,
+        _options
+      )
+      when objtype in ["Question", "Answer", "ChatMessage"] do
+    with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
+         {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
+      {:ok, activity}
+    end
+  end
+
   def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
         _options

lib/pleroma/web/common_api/common_api.ex

@@ -308,18 +308,19 @@ defmodule Pleroma.Web.CommonAPI do
          {:ok, options, choices} <- normalize_and_validate_choices(choices, object) do
       answer_activities =
         Enum.map(choices, fn index ->
-          answer_data = make_answer_data(user, object, Enum.at(options, index)["name"])
+          {:ok, answer_object, _meta} =
+            Builder.answer(user, object, Enum.at(options, index)["name"])
 
-          {:ok, activity} =
+          {:ok, activity_data, _meta} = Builder.create(user, answer_object, [])
-            ActivityPub.create(%{
-              to: answer_data["to"],
-              actor: user,
-              context: object.data["context"],
-              object: answer_data,
-              additional: %{"cc" => answer_data["cc"]}
-            })
 
-          activity
+          {:ok, activity, _meta} =
+            activity_data
+            |> Map.put("cc", answer_object["cc"])
+            |> Map.put("context", answer_object["context"])
+            |> Pipeline.common_pipeline(local: true)
+
+          # TODO: Do preload of Pleroma.Object in Pipeline
+          Activity.normalize(activity.data)
         end)
 
       object = Object.get_cached_by_ap_id(object.data["id"])
@@ -340,8 +341,13 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  defp get_options_and_max_count(%{data: %{"anyOf" => any_of}}), do: {any_of, Enum.count(any_of)}
+  defp get_options_and_max_count(%{data: %{"anyOf" => any_of}})
-  defp get_options_and_max_count(%{data: %{"oneOf" => one_of}}), do: {one_of, 1}
+       when is_list(any_of) and any_of != [],
+       do: {any_of, Enum.count(any_of)}
+
+  defp get_options_and_max_count(%{data: %{"oneOf" => one_of}})
+       when is_list(one_of) and one_of != [],
+       do: {one_of, 1}
 
   defp normalize_and_validate_choices(choices, object) do
     choices = Enum.map(choices, fn i -> if is_binary(i), do: String.to_integer(i), else: i end)

lib/pleroma/web/common_api/utils.ex

@@ -548,17 +548,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     end
   end
 
-  def make_answer_data(%User{ap_id: ap_id}, object, name) do
-    %{
-      "type" => "Answer",
-      "actor" => ap_id,
-      "cc" => [object.data["actor"]],
-      "to" => [],
-      "name" => name,
-      "inReplyTo" => object.data["id"]
-    }
-  end
-
   def validate_character_limit("" = _full_payload, [] = _attachments) do
     {:error, dgettext("errors", "Cannot post an empty status without attachments")}
   end

lib/pleroma/web/mastodon_api/views/filter_view.ex

@@ -25,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterView do
       context: filter.context,
       expires_at: expires_at,
       irreversible: filter.hide,
-      whole_word: false
+      whole_word: filter.whole_word
     }
   end
 end

lib/pleroma/web/mastodon_api/views/poll_view.ex

@@ -28,10 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
 
   def render("show.json", %{object: object} = params) do
     case object.data do
-      %{"anyOf" => options} when is_list(options) ->
+      %{"anyOf" => [_ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options}))
 
-      %{"oneOf" => options} when is_list(options) ->
+      %{"oneOf" => [_ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options}))
 
       _ ->
@@ -40,14 +40,12 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
   end
 
   defp end_time_and_expired(object) do
-    case object.data["closed"] || object.data["endTime"] do
+    if object.data["closed"] do
-      end_time when is_binary(end_time) ->
+      end_time = NaiveDateTime.from_iso8601!(object.data["closed"])
-        end_time = NaiveDateTime.from_iso8601!(end_time)
       expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt
 
       {Utils.to_masto_date(end_time), expired}
-
+    else
-      _ ->
       {nil, false}
     end
   end

lib/pleroma/web/oauth/oauth_controller.ex

@@ -76,6 +76,13 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     available_scopes = (app && app.scopes) || []
     scopes = Scopes.fetch_scopes(params, available_scopes)
 
+    scopes =
+      if scopes == [] do
+        available_scopes
+      else
+        scopes
+      end
+
     # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template
     render(conn, Authenticator.auth_template(), %{
       response_type: params["response_type"],

lib/pleroma/web/templates/layout/app.html.eex

@@ -37,7 +37,7 @@
       }
 
       a {
-        color: color: #d8a070;
+        color: #d8a070;
         text-decoration: none;
       }
 

mix.exs

@@ -229,10 +229,10 @@ defmodule Pleroma.Mixfile do
   defp version(version) do
     identifier_filter = ~r/[^0-9a-z\-]+/i
 
-    {_cmdgit, cmdgit_err} = System.cmd("sh", ["-c", "command -v git"])
+    git_available? = match?({_output, 0}, System.cmd("sh", ["-c", "command -v git"]))
 
     git_pre_release =
-      if cmdgit_err == 0 do
+      if git_available? do
         {tag, tag_err} =
           System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true)
 
@@ -258,7 +258,7 @@ defmodule Pleroma.Mixfile do
 
     # Branch name as pre-release version component, denoted with a dot
     branch_name =
-      with 0 <- cmdgit_err,
+      with true <- git_available?,
            {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]),
            branch_name <- String.trim(branch_name),
            branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name,

priv/repo/migrations/20200802170532_fix_legacy_tags.exs

@@ -0,0 +1,37 @@
+# Fix legacy tags set by AdminFE that don't align with TagPolicy MRF
+
+defmodule Pleroma.Repo.Migrations.FixLegacyTags do
+  use Ecto.Migration
+  alias Pleroma.Repo
+  alias Pleroma.User
+  import Ecto.Query
+
+  @old_new_map %{
+    "force_nsfw" => "mrf_tag:media-force-nsfw",
+    "strip_media" => "mrf_tag:media-strip",
+    "force_unlisted" => "mrf_tag:force-unlisted",
+    "sandbox" => "mrf_tag:sandbox",
+    "disable_remote_subscription" => "mrf_tag:disable-remote-subscription",
+    "disable_any_subscription" => "mrf_tag:disable-any-subscription"
+  }
+
+  def change do
+    legacy_tags = Map.keys(@old_new_map)
+
+    from(u in User, where: fragment("? && ?", u.tags, ^legacy_tags))
+    |> Repo.all()
+    |> Enum.each(fn user ->
+      fix_tags_changeset(user)
+      |> Repo.update()
+    end)
+  end
+
+  defp fix_tags_changeset(%User{tags: tags} = user) do
+    new_tags =
+      Enum.map(tags, fn tag ->
+        Map.get(@old_new_map, tag, tag)
+      end)
+
+    Ecto.Changeset.change(user, tags: new_tags)
+  end
+end

priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs

@@ -0,0 +1,19 @@
+defmodule Pleroma.Repo.Migrations.RemoveNonlocalExpirations do
+  use Ecto.Migration
+
+  def up do
+    statement = """
+    DELETE FROM
+      activity_expirations A USING activities B
+    WHERE
+      A.activity_id = B.id
+      AND B.local = false;
+    """
+
+    execute(statement)
+  end
+
+  def down do
+    :ok
+  end
+end

priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs

@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAppClientId do
+  use Ecto.Migration
+
+  def change do
+    create(unique_index(:apps, [:client_id]))
+  end
+end

test/config_test.exs

@@ -28,6 +28,34 @@ defmodule Pleroma.ConfigTest do
     assert Pleroma.Config.get([:azerty, :uiop], true) == true
   end
 
+  describe "nil values" do
+    setup do
+      Pleroma.Config.put(:lorem, nil)
+      Pleroma.Config.put(:ipsum, %{dolor: [sit: nil]})
+      Pleroma.Config.put(:dolor, sit: %{amet: nil})
+
+      on_exit(fn -> Enum.each(~w(lorem ipsum dolor)a, &Pleroma.Config.delete/1) end)
+    end
+
+    test "get/1 with an atom for nil value" do
+      assert Pleroma.Config.get(:lorem) == nil
+    end
+
+    test "get/2 with an atom for nil value" do
+      assert Pleroma.Config.get(:lorem, true) == nil
+    end
+
+    test "get/1 with a list of keys for nil value" do
+      assert Pleroma.Config.get([:ipsum, :dolor, :sit]) == nil
+      assert Pleroma.Config.get([:dolor, :sit, :amet]) == nil
+    end
+
+    test "get/2 with a list of keys for nil value" do
+      assert Pleroma.Config.get([:ipsum, :dolor, :sit], true) == nil
+      assert Pleroma.Config.get([:dolor, :sit, :amet], true) == nil
+    end
+  end
+
   test "get/1 when value is false" do
     Pleroma.Config.put([:instance, :false_test], false)
     Pleroma.Config.put([:instance, :nested], [])
@@ -89,5 +117,23 @@ defmodule Pleroma.ConfigTest do
     Pleroma.Config.put([:delete_me, :delete_me], hello: "world", world: "Hello")
     Pleroma.Config.delete([:delete_me, :delete_me, :world])
     assert Pleroma.Config.get([:delete_me, :delete_me]) == [hello: "world"]
+
+    assert Pleroma.Config.delete([:this_key_does_not_exist])
+    assert Pleroma.Config.delete([:non, :existing, :key])
+  end
+
+  test "fetch/1" do
+    Pleroma.Config.put([:lorem], :ipsum)
+    Pleroma.Config.put([:ipsum], dolor: :sit)
+
+    assert Pleroma.Config.fetch([:lorem]) == {:ok, :ipsum}
+    assert Pleroma.Config.fetch(:lorem) == {:ok, :ipsum}
+    assert Pleroma.Config.fetch([:ipsum, :dolor]) == {:ok, :sit}
+    assert Pleroma.Config.fetch([:lorem, :ipsum]) == :error
+    assert Pleroma.Config.fetch([:loremipsum]) == :error
+    assert Pleroma.Config.fetch(:loremipsum) == :error
+
+    Pleroma.Config.delete([:lorem])
+    Pleroma.Config.delete([:ipsum])
   end
 end

test/emails/mailer_test.exs

@@ -19,6 +19,7 @@ defmodule Pleroma.Emails.MailerTest do
   test "not send email when mailer is disabled" do
     Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false)
     Mailer.deliver(@email)
+    :timer.sleep(100)
 
     refute_email_sent(
       from: {"Pleroma", "noreply@example.com"},
@@ -30,6 +31,7 @@ defmodule Pleroma.Emails.MailerTest do
 
   test "send email" do
     Mailer.deliver(@email)
+    :timer.sleep(100)
 
     assert_email_sent(
       from: {"Pleroma", "noreply@example.com"},
@@ -41,6 +43,7 @@ defmodule Pleroma.Emails.MailerTest do
 
   test "perform" do
     Mailer.perform(:deliver_async, @email, [])
+    :timer.sleep(100)
 
     assert_email_sent(
       from: {"Pleroma", "noreply@example.com"},

test/fixtures/mastodon-question-activity.json

@@ -49,7 +49,6 @@
       "en": "<p>Why is Tenshi eating a corndog so cute?</p>"
     },
     "endTime": "2019-05-11T09:03:36Z",
-    "closed": "2019-05-11T09:03:36Z",
     "attachment": [],
     "tag": [],
     "replies": {

test/fixtures/tesla_mock/poll_attachment.json

@@ -0,0 +1,99 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://patch.cx/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "actor": "https://patch.cx/users/rin",
+  "anyOf": [],
+  "attachment": [
+    {
+      "mediaType": "image/jpeg",
+      "name": "screenshot_mpv:Totoro@01:18:44.345.jpg",
+      "type": "Document",
+      "url": "https://shitposter.club/media/3bb4c4d402f8fdcc7f80963c3d7cf6f10f936897fd374922ade33199d2f86d87.jpg?name=screenshot_mpv%3ATotoro%4001%3A18%3A44.345.jpg"
+    }
+  ],
+  "attributedTo": "https://patch.cx/users/rin",
+  "cc": [
+    "https://patch.cx/users/rin/followers"
+  ],
+  "closed": "2020-06-19T23:22:02.754678Z",
+  "content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"9vwjTNzEWEM1TfkBGq\" href=\"https://mastodon.sdf.org/users/rinpatch\" rel=\"ugc\">@<span>rinpatch</span></a></span>",
+  "closed": "2019-09-19T00:32:36.785333",
+  "content": "can you vote on this poll?",
+  "id": "https://patch.cx/objects/tesla_mock/poll_attachment",
+  "oneOf": [
+    {
+      "name": "a",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "A",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "Aa",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "AA",
+      "replies": {
+        "totalItems": 0,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "AAa",
+      "replies": {
+        "totalItems": 1,
+        "type": "Collection"
+      },
+      "type": "Note"
+    },
+    {
+      "name": "AAA",
+      "replies": {
+        "totalItems": 3,
+        "type": "Collection"
+      },
+      "type": "Note"
+    }
+  ],
+  "published": "2020-06-19T23:12:02.786113Z",
+  "sensitive": false,
+  "summary": "",
+  "tag": [
+    {
+      "href": "https://mastodon.sdf.org/users/rinpatch",
+      "name": "@rinpatch@mastodon.sdf.org",
+      "type": "Mention"
+    }
+  ],
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public",
+    "https://mastodon.sdf.org/users/rinpatch"
+  ],
+  "type": "Question",
+  "voters": [
+    "https://shitposter.club/users/moonman",
+    "https://skippers-bin.com/users/7v1w1r8ce6",
+    "https://mastodon.sdf.org/users/rinpatch",
+    "https://mastodon.social/users/emelie"
+  ]
+}

test/migrations/20200802170532_fix_legacy_tags_test.exs

@@ -0,0 +1,24 @@
+defmodule Pleroma.Repo.Migrations.FixLegacyTagsTest do
+  alias Pleroma.User
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  import Pleroma.Tests.Helpers
+
+  setup_all do: require_migration("20200802170532_fix_legacy_tags")
+
+  test "change/0 converts legacy user tags into correct values", %{migration: migration} do
+    user = insert(:user, tags: ["force_nsfw", "force_unlisted", "verified"])
+    user2 = insert(:user)
+
+    assert :ok == migration.change()
+
+    fixed_user = User.get_by_id(user.id)
+    fixed_user2 = User.get_by_id(user2.id)
+
+    assert fixed_user.tags == ["mrf_tag:media-force-nsfw", "mrf_tag:force-unlisted", "verified"]
+    assert fixed_user2.tags == []
+
+    # user2 should not have been updated
+    assert fixed_user2.updated_at == fixed_user2.inserted_at
+  end
+end

test/object/fetcher_test.exs

@@ -177,6 +177,13 @@ defmodule Pleroma.Object.FetcherTest do
                  "https://mastodon.example.org/users/userisgone404"
                )
     end
+
+    test "it can fetch pleroma polls with attachments" do
+      {:ok, object} =
+        Fetcher.fetch_object_from_id("https://patch.cx/objects/tesla_mock/poll_attachment")
+
+      assert object
+    end
   end
 
   describe "pruning" do

test/support/helpers.ex

@@ -17,9 +17,19 @@ defmodule Pleroma.Tests.Helpers do
 
   defmacro clear_config(config_path, do: yield) do
     quote do
-      initial_setting = Config.get(unquote(config_path))
+      initial_setting = Config.fetch(unquote(config_path))
       unquote(yield)
-      on_exit(fn -> Config.put(unquote(config_path), initial_setting) end)
+
+      on_exit(fn ->
+        case initial_setting do
+          :error ->
+            Config.delete(unquote(config_path))
+
+          {:ok, value} ->
+            Config.put(unquote(config_path), value)
+        end
+      end)
+
       :ok
     end
   end

test/support/http_request_mock.ex

@@ -82,6 +82,14 @@ defmodule HttpRequestMock do
      }}
   end
 
+  def get("https://patch.cx/objects/tesla_mock/poll_attachment", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/poll_attachment.json")
+     }}
+  end
+
   def get(
         "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/emelie",
         _,

test/upload/filter/exiftool_test.exs

@@ -7,6 +7,8 @@ defmodule Pleroma.Upload.Filter.ExiftoolTest do
   alias Pleroma.Upload.Filter
 
   test "apply exiftool filter" do
+    assert Pleroma.Utils.command_available?("exiftool")
+
     File.cp!(
       "test/fixtures/DSCN0010.jpg",
       "test/fixtures/DSCN0010_tmp.jpg"

test/web/activity_pub/object_validators/delete_validation_test.exs

@@ -87,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidationTest do
 
       {:error, cng} = ObjectValidator.validate(invalid_other_actor, [])
 
-      assert {:actor, {"is not allowed to delete object", []}} in cng.errors
+      assert {:actor, {"is not allowed to modify object", []}} in cng.errors
     end
 
     test "it's valid if the actor of the object is a local superuser",

test/web/activity_pub/transmogrifier/answer_handling_test.exs

@@ -0,0 +1,78 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.AnswerHandlingTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.CommonAPI
+
+  import Pleroma.Factory
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  test "incoming, rewrites Note to Answer and increments vote counters" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "suya...",
+        poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
+      })
+
+    object = Object.normalize(activity)
+
+    data =
+      File.read!("test/fixtures/mastodon-vote.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["to"], user.ap_id)
+      |> Kernel.put_in(["object", "inReplyTo"], object.data["id"])
+      |> Kernel.put_in(["object", "to"], user.ap_id)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+    answer_object = Object.normalize(activity)
+    assert answer_object.data["type"] == "Answer"
+    assert answer_object.data["inReplyTo"] == object.data["id"]
+
+    new_object = Object.get_by_ap_id(object.data["id"])
+    assert new_object.data["replies_count"] == object.data["replies_count"]
+
+    assert Enum.any?(
+             new_object.data["oneOf"],
+             fn
+               %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true
+               _ -> false
+             end
+           )
+  end
+
+  test "outgoing, rewrites Answer to Note" do
+    user = insert(:user)
+
+    {:ok, poll_activity} =
+      CommonAPI.post(user, %{
+        status: "suya...",
+        poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
+      })
+
+    poll_object = Object.normalize(poll_activity)
+    # TODO: Replace with CommonAPI vote creation when implemented
+    data =
+      File.read!("test/fixtures/mastodon-vote.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["to"], user.ap_id)
+      |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"])
+      |> Kernel.put_in(["object", "to"], user.ap_id)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+    {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
+
+    assert data["object"]["type"] == "Note"
+  end
+end

test/web/activity_pub/transmogrifier/question_handling_test.exs

@@ -0,0 +1,123 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.CommonAPI
+
+  import Pleroma.Factory
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  test "Mastodon Question activity" do
+    data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!()
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    object = Object.normalize(activity, false)
+
+    assert object.data["closed"] == "2019-05-11T09:03:36Z"
+
+    assert object.data["context"] == activity.data["context"]
+
+    assert object.data["context"] ==
+             "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation"
+
+    assert object.data["context_id"]
+
+    assert object.data["anyOf"] == []
+
+    assert Enum.sort(object.data["oneOf"]) ==
+             Enum.sort([
+               %{
+                 "name" => "25 char limit is dumb",
+                 "replies" => %{"totalItems" => 0, "type" => "Collection"},
+                 "type" => "Note"
+               },
+               %{
+                 "name" => "Dunno",
+                 "replies" => %{"totalItems" => 0, "type" => "Collection"},
+                 "type" => "Note"
+               },
+               %{
+                 "name" => "Everyone knows that!",
+                 "replies" => %{"totalItems" => 1, "type" => "Collection"},
+                 "type" => "Note"
+               },
+               %{
+                 "name" => "I can't even fit a funny",
+                 "replies" => %{"totalItems" => 1, "type" => "Collection"},
+                 "type" => "Note"
+               }
+             ])
+
+    user = insert(:user)
+
+    {:ok, reply_activity} = CommonAPI.post(user, %{status: "hewwo", in_reply_to_id: activity.id})
+
+    reply_object = Object.normalize(reply_activity, false)
+
+    assert reply_object.data["context"] == object.data["context"]
+    assert reply_object.data["context_id"] == object.data["context_id"]
+  end
+
+  test "Mastodon Question activity with HTML tags in plaintext" do
+    options = [
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\">",
+        "replies" => %{"totalItems" => 0, "type" => "Collection"}
+      },
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\"/>",
+        "replies" => %{"totalItems" => 0, "type" => "Collection"}
+      },
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\" />",
+        "replies" => %{"totalItems" => 1, "type" => "Collection"}
+      },
+      %{
+        "type" => "Note",
+        "name" => "<input type=\"date\"></input>",
+        "replies" => %{"totalItems" => 1, "type" => "Collection"}
+      }
+    ]
+
+    data =
+      File.read!("test/fixtures/mastodon-question-activity.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["object", "oneOf"], options)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+    object = Object.normalize(activity, false)
+
+    assert Enum.sort(object.data["oneOf"]) == Enum.sort(options)
+  end
+
+  test "returns an error if received a second time" do
+    data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!()
+
+    assert {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert {:error, {:validate_object, {:error, _}}} = Transmogrifier.handle_incoming(data)
+  end
+
+  test "accepts a Question with no content" do
+    data =
+      File.read!("test/fixtures/mastodon-question-activity.json")
+      |> Poison.decode!()
+      |> Kernel.put_in(["object", "content"], "")
+
+    assert {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data)
+  end
+end

test/web/activity_pub/transmogrifier_test.exs

@@ -225,23 +225,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert Enum.at(object.data["tag"], 2) == "moo"
     end
 
-    test "it works for incoming questions" do
-      data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!()
-
-      {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-
-      object = Object.normalize(activity)
-
-      assert Enum.all?(object.data["oneOf"], fn choice ->
-               choice["name"] in [
-                 "Dunno",
-                 "Everyone knows that!",
-                 "25 char limit is dumb",
-                 "I can't even fit a funny"
-               ]
-             end)
-    end
-
     test "it works for incoming listens" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
@@ -271,38 +254,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert object.data["length"] == 180_000
     end
 
-    test "it rewrites Note votes to Answers and increments vote counters on question activities" do
-      user = insert(:user)
-
-      {:ok, activity} =
-        CommonAPI.post(user, %{
-          status: "suya...",
-          poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
-        })
-
-      object = Object.normalize(activity)
-
-      data =
-        File.read!("test/fixtures/mastodon-vote.json")
-        |> Poison.decode!()
-        |> Kernel.put_in(["to"], user.ap_id)
-        |> Kernel.put_in(["object", "inReplyTo"], object.data["id"])
-        |> Kernel.put_in(["object", "to"], user.ap_id)
-
-      {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-      answer_object = Object.normalize(activity)
-      assert answer_object.data["type"] == "Answer"
-      object = Object.get_by_ap_id(object.data["id"])
-
-      assert Enum.any?(
-               object.data["oneOf"],
-               fn
-                 %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true
-                 _ -> false
-               end
-             )
-    end
-
     test "it works for incoming notices with contentMap" do
       data =
         File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!()
@@ -677,7 +628,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                    %{
                      "href" =>
                        "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4",
-                     "mediaType" => "video/mp4"
+                     "mediaType" => "video/mp4",
+                     "type" => "Link"
                    }
                  ]
                }
@@ -696,7 +648,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                    %{
                      "href" =>
                        "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4",
-                     "mediaType" => "video/mp4"
+                     "mediaType" => "video/mp4",
+                     "type" => "Link"
                    }
                  ]
                }
@@ -1269,30 +1222,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
     end
   end
 
-  test "Rewrites Answers to Notes" do
-    user = insert(:user)
-
-    {:ok, poll_activity} =
-      CommonAPI.post(user, %{
-        status: "suya...",
-        poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
-      })
-
-    poll_object = Object.normalize(poll_activity)
-    # TODO: Replace with CommonAPI vote creation when implemented
-    data =
-      File.read!("test/fixtures/mastodon-vote.json")
-      |> Poison.decode!()
-      |> Kernel.put_in(["to"], user.ap_id)
-      |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"])
-      |> Kernel.put_in(["object", "to"], user.ap_id)
-
-    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-    {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
-
-    assert data["object"]["type"] == "Note"
-  end
-
   describe "fix_explicit_addressing" do
     setup do
       user = insert(:user)
@@ -1540,8 +1469,13 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                "attachment" => [
                  %{
                    "mediaType" => "video/mp4",
+                   "type" => "Document",
                    "url" => [
-                     %{"href" => "https://peertube.moe/stat-480.mp4", "mediaType" => "video/mp4"}
+                     %{
+                       "href" => "https://peertube.moe/stat-480.mp4",
+                       "mediaType" => "video/mp4",
+                       "type" => "Link"
+                     }
                    ]
                  }
                ]
@@ -1558,14 +1492,24 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
                "attachment" => [
                  %{
                    "mediaType" => "video/mp4",
+                   "type" => "Document",
                    "url" => [
-                     %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"}
+                     %{
+                       "href" => "https://pe.er/stat-480.mp4",
+                       "mediaType" => "video/mp4",
+                       "type" => "Link"
+                     }
                    ]
                  },
                  %{
                    "mediaType" => "video/mp4",
+                   "type" => "Document",
                    "url" => [
-                     %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"}
+                     %{
+                       "href" => "https://pe.er/stat-480.mp4",
+                       "mediaType" => "video/mp4",
+                       "type" => "Link"
+                     }
                    ]
                  }
                ]

test/web/mastodon_api/controllers/filter_controller_test.exs

@@ -64,11 +64,13 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
   test "get a filter" do
     %{user: user, conn: conn} = oauth_access(["read:filters"])
 
+    # check whole_word false
     query = %Pleroma.Filter{
       user_id: user.id,
       filter_id: 2,
       phrase: "knight",
-      context: ["home"]
+      context: ["home"],
+      whole_word: false
     }
 
     {:ok, filter} = Pleroma.Filter.create(query)
@@ -76,6 +78,25 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
     conn = get(conn, "/api/v1/filters/#{filter.filter_id}")
 
     assert response = json_response_and_validate_schema(conn, 200)
+    assert response["whole_word"] == false
+
+    # check whole_word true
+    %{user: user, conn: conn} = oauth_access(["read:filters"])
+
+    query = %Pleroma.Filter{
+      user_id: user.id,
+      filter_id: 3,
+      phrase: "knight",
+      context: ["home"],
+      whole_word: true
+    }
+
+    {:ok, filter} = Pleroma.Filter.create(query)
+
+    conn = get(conn, "/api/v1/filters/#{filter.filter_id}")
+
+    assert response = json_response_and_validate_schema(conn, 200)
+    assert response["whole_word"] == true
   end
 
   test "update a filter" do
@@ -86,7 +107,8 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
       filter_id: 2,
       phrase: "knight",
       context: ["home"],
-      hide: true
+      hide: true,
+      whole_word: true
     }
 
     {:ok, _filter} = Pleroma.Filter.create(query)
@@ -108,6 +130,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
     assert response["phrase"] == new.phrase
     assert response["context"] == new.context
     assert response["irreversible"] == true
+    assert response["whole_word"] == true
   end
 
   test "delete a filter" do

test/web/mastodon_api/views/poll_view_test.exs

@@ -135,4 +135,33 @@ defmodule Pleroma.Web.MastodonAPI.PollViewTest do
     assert result[:expires_at] == nil
     assert result[:expired] == false
   end
+
+  test "doesn't strips HTML tags" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "What's with the smug face?",
+        poll: %{
+          options: [
+            "<input type=\"date\">",
+            "<input type=\"date\" >",
+            "<input type=\"date\"/>",
+            "<input type=\"date\"></input>"
+          ],
+          expires_in: 20
+        }
+      })
+
+    object = Object.normalize(activity)
+
+    assert %{
+             options: [
+               %{title: "<input type=\"date\">", votes_count: 0},
+               %{title: "<input type=\"date\" >", votes_count: 0},
+               %{title: "<input type=\"date\"/>", votes_count: 0},
+               %{title: "<input type=\"date\"></input>", votes_count: 0}
+             ]
+           } = PollView.render("show.json", %{object: object})
+  end
 end

test/web/oauth/app_test.exs

@@ -29,5 +29,16 @@ defmodule Pleroma.Web.OAuth.AppTest do
       assert exist_app.id == app.id
       assert exist_app.scopes == ["read", "write", "follow", "push"]
     end
+
+    test "has unique client_id" do
+      insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop")
+
+      error =
+        catch_error(insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop"))
+
+      assert %Ecto.ConstraintError{} = error
+      assert error.constraint == "apps_client_id_index"
+      assert error.type == :unique
+    end
   end
 end