Merge branch 'issue/1276-2' into 'develop'

[#1276] added an endpoint for getting unread notification count

See merge request pleroma/pleroma!2392

CHANGELOG.md

@@ -156,6 +156,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: `pleroma.thread_muted` to the Status entity
 - Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
 - Mastodon API, streaming: Add `pleroma.direct_conversation_id` to the `conversation` stream event payload.
+- Mastodon API: Add `pleroma.unread_count` to the Marker entity
 - Admin API: Render whole status in grouped reports
 - Mastodon API: User timelines will now respect blocks, unless you are getting the user timeline of somebody you blocked (which would be empty otherwise).
 - Mastodon API: Favoriting / Repeating a post multiple times will now return the identical response every time. Before, executing that action twice would return an error ("already favorited") on the second try.

docs/API/differences_in_mastoapi_responses.md

@@ -61,6 +61,7 @@ Has these additional fields under the `pleroma` object:
 - `deactivated`: boolean, true when the user is deactivated
 - `allow_following_move`: boolean, true when the user allows automatically follow moved following accounts
 - `unread_conversation_count`: The count of unread conversations. Only returned to the account owner.
+- `unread_notifications_count`: The count of unread notifications. Only returned to the account owner.
 
 ### Source
 
@@ -218,3 +219,9 @@ Has theses additional parameters (which are the same as in Pleroma-API):
 - `pleroma.metadata.features`: A list of supported features
 - `pleroma.metadata.federation`: The federation restrictions of this instance
 - `vapid_public_key`: The public key needed for push messages
+
+## Markers
+
+Has these additional fields under the `pleroma` object:
+
+- `unread_count`: contains number unread notifications

lib/pleroma/marker.ex

@@ -9,24 +9,34 @@ defmodule Pleroma.Marker do
   import Ecto.Query
 
   alias Ecto.Multi
+  alias Pleroma.Notification
   alias Pleroma.Repo
   alias Pleroma.User
+  alias __MODULE__
 
   @timelines ["notifications"]
+  @type t :: %__MODULE__{}
 
   schema "markers" do
     field(:last_read_id, :string, default: "")
     field(:timeline, :string, default: "")
     field(:lock_version, :integer, default: 0)
+    field(:unread_count, :integer, default: 0, virtual: true)
 
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     timestamps()
   end
 
+  @doc "Gets markers by user and timeline."
+  @spec get_markers(User.t(), list(String)) :: list(t())
   def get_markers(user, timelines \\ []) do
-    Repo.all(get_query(user, timelines))
+    user
+    |> get_query(timelines)
+    |> unread_count_query()
+    |> Repo.all()
   end
 
+  @spec upsert(User.t(), map()) :: {:ok | :error, any()}
   def upsert(%User{} = user, attrs) do
     attrs
     |> Map.take(@timelines)
@@ -45,6 +55,27 @@ defmodule Pleroma.Marker do
     |> Repo.transaction()
   end
 
+  @spec multi_set_last_read_id(Multi.t(), User.t(), String.t()) :: Multi.t()
+  def multi_set_last_read_id(multi, %User{} = user, "notifications") do
+    multi
+    |> Multi.run(:counters, fn _repo, _changes ->
+      {:ok, %{last_read_id: Repo.one(Notification.last_read_query(user))}}
+    end)
+    |> Multi.insert(
+      :marker,
+      fn %{counters: attrs} ->
+        %Marker{timeline: "notifications", user_id: user.id}
+        |> struct(attrs)
+        |> Ecto.Changeset.change()
+      end,
+      returning: true,
+      on_conflict: {:replace, [:last_read_id]},
+      conflict_target: [:user_id, :timeline]
+    )
+  end
+
+  def multi_set_last_read_id(multi, _, _), do: multi
+
   defp get_marker(user, timeline) do
     case Repo.find_resource(get_query(user, timeline)) do
       {:ok, marker} -> %__MODULE__{marker | user: user}
@@ -71,4 +102,16 @@ defmodule Pleroma.Marker do
     |> by_user_id(user.id)
     |> by_timeline(timelines)
   end
+
+  defp unread_count_query(query) do
+    from(
+      q in query,
+      left_join: n in "notifications",
+      on: n.user_id == q.user_id and n.seen == false,
+      group_by: [:id],
+      select_merge: %{
+        unread_count: fragment("count(?)", n.id)
+      }
+    )
+  end
 end

lib/pleroma/notification.ex

@@ -5,8 +5,10 @@
 defmodule Pleroma.Notification do
   use Ecto.Schema
 
+  alias Ecto.Multi
   alias Pleroma.Activity
   alias Pleroma.FollowingRelationship
+  alias Pleroma.Marker
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Pagination
@@ -34,11 +36,30 @@ defmodule Pleroma.Notification do
     timestamps()
   end
 
+  @spec unread_notifications_count(User.t()) :: integer()
+  def unread_notifications_count(%User{id: user_id}) do
+    from(q in __MODULE__,
+      where: q.user_id == ^user_id and q.seen == false
+    )
+    |> Repo.aggregate(:count, :id)
+  end
+
   def changeset(%Notification{} = notification, attrs) do
     notification
     |> cast(attrs, [:seen])
   end
 
+  @spec last_read_query(User.t()) :: Ecto.Queryable.t()
+  def last_read_query(user) do
+    from(q in Pleroma.Notification,
+      where: q.user_id == ^user.id,
+      where: q.seen == true,
+      select: type(q.id, :string),
+      limit: 1,
+      order_by: [desc: :id]
+    )
+  end
+
   defp for_user_query_ap_id_opts(user, opts) do
     ap_id_relationships =
       [:block] ++
@@ -185,25 +206,23 @@ defmodule Pleroma.Notification do
     |> Repo.all()
   end
 
-  def set_read_up_to(%{id: user_id} = _user, id) do
+  def set_read_up_to(%{id: user_id} = user, id) do
     query =
       from(
         n in Notification,
         where: n.user_id == ^user_id,
         where: n.id <= ^id,
         where: n.seen == false,
-        update: [
-          set: [
-            seen: true,
-            updated_at: ^NaiveDateTime.utc_now()
-          ]
-        ],
         # Ideally we would preload object and activities here
         # but Ecto does not support preloads in update_all
         select: n.id
       )
 
-    {_, notification_ids} = Repo.update_all(query, [])
+    {:ok, %{ids: {_, notification_ids}}} =
+      Multi.new()
+      |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
+      |> Marker.multi_set_last_read_id(user, "notifications")
+      |> Repo.transaction()
 
     Notification
     |> where([n], n.id in ^notification_ids)
@@ -220,11 +239,18 @@ defmodule Pleroma.Notification do
     |> Repo.all()
   end
 
+  @spec read_one(User.t(), String.t()) ::
+          {:ok, Notification.t()} | {:error, Ecto.Changeset.t()} | nil
   def read_one(%User{} = user, notification_id) do
     with {:ok, %Notification{} = notification} <- get(user, notification_id) do
-      notification
+      Multi.new()
-      |> changeset(%{seen: true})
+      |> Multi.update(:update, changeset(notification, %{seen: true}))
-      |> Repo.update()
+      |> Marker.multi_set_last_read_id(user, "notifications")
+      |> Repo.transaction()
+      |> case do
+        {:ok, %{update: notification}} -> {:ok, notification}
+        {:error, :update, changeset, _} -> {:error, changeset}
+      end
     end
   end
 
@@ -316,8 +342,11 @@ defmodule Pleroma.Notification do
   # TODO move to sql, too.
   def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do
     unless skip?(activity, user) do
-      notification = %Notification{user_id: user.id, activity: activity}
+      {:ok, %{notification: notification}} =
-      {:ok, notification} = Repo.insert(notification)
+        Multi.new()
+        |> Multi.insert(:notification, %Notification{user_id: user.id, activity: activity})
+        |> Marker.multi_set_last_read_id(user, "notifications")
+        |> Repo.transaction()
 
       if do_send do
         Streamer.stream(["user", "user:notification"], notification)

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -36,9 +36,11 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   end
 
   def render("show.json", %{user: user} = opts) do
-    if User.visible_for?(user, opts[:for]),
+    if User.visible_for?(user, opts[:for]) do
-      do: do_render("show.json", opts),
+      do_render("show.json", opts)
-      else: %{}
+    else
+      %{}
+    end
   end
 
   def render("mention.json", %{user: user}) do
@@ -221,7 +223,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       fields: user.fields,
       bot: bot,
       source: %{
-        note: (user.bio || "") |> String.replace(~r(<br */?>), "\n") |> Pleroma.HTML.strip_tags(),
+        note: prepare_user_bio(user),
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
@@ -253,8 +255,17 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     |> maybe_put_follow_requests_count(user, opts[:for])
     |> maybe_put_allow_following_move(user, opts[:for])
     |> maybe_put_unread_conversation_count(user, opts[:for])
+    |> maybe_put_unread_notification_count(user, opts[:for])
   end
 
+  defp prepare_user_bio(%User{bio: ""}), do: ""
+
+  defp prepare_user_bio(%User{bio: bio}) when is_binary(bio) do
+    bio |> String.replace(~r(<br */?>), "\n") |> Pleroma.HTML.strip_tags()
+  end
+
+  defp prepare_user_bio(_), do: ""
+
   defp username_from_nickname(string) when is_binary(string) do
     hd(String.split(string, "@"))
   end
@@ -350,6 +361,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
 
   defp maybe_put_unread_conversation_count(data, _, _), do: data
 
+  defp maybe_put_unread_notification_count(data, %User{id: user_id}, %User{id: user_id} = user) do
+    Kernel.put_in(
+      data,
+      [:pleroma, :unread_notifications_count],
+      Pleroma.Notification.unread_notifications_count(user)
+    )
+  end
+
+  defp maybe_put_unread_notification_count(data, _, _), do: data
+
   defp image_url(%{"url" => [%{"href" => href} | _]}), do: href
   defp image_url(_), do: nil
 end

lib/pleroma/web/mastodon_api/views/marker_view.ex

@@ -11,7 +11,10 @@ defmodule Pleroma.Web.MastodonAPI.MarkerView do
        %{
          last_read_id: m.last_read_id,
          version: m.lock_version,
-         updated_at: NaiveDateTime.to_iso8601(m.updated_at)
+         updated_at: NaiveDateTime.to_iso8601(m.updated_at),
+         pleroma: %{
+           unread_count: m.unread_count
+         }
        }}
     end)
   end

priv/repo/migrations/20200415181818_update_markers.exs

@@ -0,0 +1,40 @@
+defmodule Pleroma.Repo.Migrations.UpdateMarkers do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  def up do
+    update_markers()
+  end
+
+  def down do
+    :ok
+  end
+
+  defp update_markers do
+    now = NaiveDateTime.utc_now()
+
+    markers_attrs =
+      from(q in "notifications",
+        select: %{
+          timeline: "notifications",
+          user_id: q.user_id,
+          last_read_id:
+            type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string)
+        },
+        group_by: [q.user_id]
+      )
+      |> Repo.all()
+      |> Enum.map(fn %{last_read_id: last_read_id} = attrs ->
+        attrs
+        |> Map.put(:last_read_id, last_read_id || "")
+        |> Map.put_new(:inserted_at, now)
+        |> Map.put_new(:updated_at, now)
+      end)
+
+    Repo.insert_all("markers", markers_attrs,
+      on_conflict: {:replace, [:last_read_id]},
+      conflict_target: [:user_id, :timeline]
+    )
+  end
+end

test/marker_test.exs

@@ -8,12 +8,39 @@ defmodule Pleroma.MarkerTest do
 
   import Pleroma.Factory
 
+  describe "multi_set_unread_count/3" do
+    test "returns multi" do
+      user = insert(:user)
+
+      assert %Ecto.Multi{
+               operations: [marker: {:run, _}, counters: {:run, _}]
+             } =
+               Marker.multi_set_last_read_id(
+                 Ecto.Multi.new(),
+                 user,
+                 "notifications"
+               )
+    end
+
+    test "return empty multi" do
+      user = insert(:user)
+      multi = Ecto.Multi.new()
+      assert Marker.multi_set_last_read_id(multi, user, "home") == multi
+    end
+  end
+
   describe "get_markers/2" do
     test "returns user markers" do
       user = insert(:user)
       marker = insert(:marker, user: user)
+      insert(:notification, user: user)
+      insert(:notification, user: user)
       insert(:marker, timeline: "home", user: user)
-      assert Marker.get_markers(user, ["notifications"]) == [refresh_record(marker)]
+
+      assert Marker.get_markers(
+               user,
+               ["notifications"]
+             ) == [%Marker{refresh_record(marker) | unread_count: 2}]
     end
   end
 

test/notification_test.exs

@@ -47,6 +47,9 @@ defmodule Pleroma.NotificationTest do
       assert notified_ids == [other_user.id, third_user.id]
       assert notification.activity_id == activity.id
       assert other_notification.activity_id == activity.id
+
+      assert [%Pleroma.Marker{unread_count: 2}] =
+               Pleroma.Marker.get_markers(other_user, ["notifications"])
     end
 
     test "it creates a notification for subscribed users" do
@@ -466,6 +469,16 @@ defmodule Pleroma.NotificationTest do
       assert n1.seen == true
       assert n2.seen == true
       assert n3.seen == false
+
+      assert %Pleroma.Marker{} =
+               m =
+               Pleroma.Repo.get_by(
+                 Pleroma.Marker,
+                 user_id: other_user.id,
+                 timeline: "notifications"
+               )
+
+      assert m.last_read_id == to_string(n2.id)
     end
   end
 

test/web/mastodon_api/controllers/account_controller_test.exs

@@ -1196,12 +1196,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do
   describe "verify_credentials" do
     test "verify_credentials" do
       %{user: user, conn: conn} = oauth_access(["read:accounts"])
+      [notification | _] = insert_list(7, :notification, user: user)
+      Pleroma.Notification.set_read_up_to(user, notification.id)
       conn = get(conn, "/api/v1/accounts/verify_credentials")
 
       response = json_response_and_validate_schema(conn, 200)
 
       assert %{"id" => id, "source" => %{"privacy" => "public"}} = response
       assert response["pleroma"]["chat_token"]
+      assert response["pleroma"]["unread_notifications_count"] == 6
       assert id == to_string(user.id)
     end
 

test/web/mastodon_api/controllers/marker_controller_test.exs

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do
     test "gets markers with correct scopes", %{conn: conn} do
       user = insert(:user)
       token = insert(:oauth_token, user: user, scopes: ["read:statuses"])
+      insert_list(7, :notification, user: user)
 
       {:ok, %{"notifications" => marker}} =
         Pleroma.Marker.upsert(
@@ -29,7 +30,8 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do
                "notifications" => %{
                  "last_read_id" => "69420",
                  "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at),
-                 "version" => 0
+                 "version" => 0,
+                 "pleroma" => %{"unread_count" => 7}
                }
              }
     end
@@ -71,7 +73,8 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do
                "notifications" => %{
                  "last_read_id" => "69420",
                  "updated_at" => _,
-                 "version" => 0
+                 "version" => 0,
+                 "pleroma" => %{"unread_count" => 0}
                }
              } = response
     end
@@ -101,7 +104,8 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do
                "notifications" => %{
                  "last_read_id" => "69888",
                  "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at),
-                 "version" => 0
+                 "version" => 0,
+                 "pleroma" => %{"unread_count" => 0}
                }
              }
     end

test/web/mastodon_api/views/account_view_test.exs

@@ -466,6 +466,24 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
                :unread_conversation_count
              ] == 1
     end
+
+    test "shows unread_count only to the account owner" do
+      user = insert(:user)
+      insert_list(7, :notification, user: user)
+      other_user = insert(:user)
+
+      user = User.get_cached_by_ap_id(user.ap_id)
+
+      assert AccountView.render(
+               "show.json",
+               %{user: user, for: other_user}
+             )[:pleroma][:unread_notifications_count] == nil
+
+      assert AccountView.render(
+               "show.json",
+               %{user: user, for: user}
+             )[:pleroma][:unread_notifications_count] == 7
+    end
   end
 
   describe "follow requests counter" do

test/web/mastodon_api/views/marker_view_test.exs

@@ -8,19 +8,21 @@ defmodule Pleroma.Web.MastodonAPI.MarkerViewTest do
   import Pleroma.Factory
 
   test "returns markers" do
-    marker1 = insert(:marker, timeline: "notifications", last_read_id: "17")
+    marker1 = insert(:marker, timeline: "notifications", last_read_id: "17", unread_count: 5)
     marker2 = insert(:marker, timeline: "home", last_read_id: "42")
 
     assert MarkerView.render("markers.json", %{markers: [marker1, marker2]}) == %{
              "home" => %{
                last_read_id: "42",
                updated_at: NaiveDateTime.to_iso8601(marker2.updated_at),
-               version: 0
+               version: 0,
+               pleroma: %{unread_count: 0}
              },
              "notifications" => %{
                last_read_id: "17",
                updated_at: NaiveDateTime.to_iso8601(marker1.updated_at),
-               version: 0
+               version: 0,
+               pleroma: %{unread_count: 5}
              }
            }
   end