diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 4c92a58c7..211b872f9 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -72,6 +72,11 @@ defmodule Pleroma.Chat do
     |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
   end
 
+  def get_by_id(id) do
+    __MODULE__
+    |> Repo.get(id)
+  end
+
   def get(user_id, recipient) do
     __MODULE__
     |> Repo.get_by(user_id: user_id, recipient: recipient)
diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat_message_reference.ex
new file mode 100644
index 000000000..e9ca3dfe8
--- /dev/null
+++ b/lib/pleroma/chat_message_reference.ex
@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ChatMessageReference do
+  @moduledoc """
+  A reference that builds a relation between an AP chat message that a user can see and whether it has been seen
+  by them, or should be displayed to them. Used to build the chat view that is presented to the user.
+  """
+
+  use Ecto.Schema
+
+  alias Pleroma.Chat
+  alias Pleroma.Object
+  alias Pleroma.Repo
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
+
+  schema "chat_message_references" do
+    belongs_to(:object, Object)
+    belongs_to(:chat, Chat)
+
+    field(:seen, :boolean, default: false)
+
+    timestamps()
+  end
+
+  def changeset(struct, params) do
+    struct
+    |> cast(params, [:object_id, :chat_id, :seen])
+    |> validate_required([:object_id, :chat_id, :seen])
+  end
+
+  def get_by_id(id) do
+    __MODULE__
+    |> Repo.get(id)
+    |> Repo.preload(:object)
+  end
+
+  def delete(cm_ref) do
+    cm_ref
+    |> Repo.delete()
+  end
+
+  def delete_for_object(%{id: object_id}) do
+    from(cr in __MODULE__,
+      where: cr.object_id == ^object_id
+    )
+    |> Repo.delete_all()
+  end
+
+  def for_chat_and_object(%{id: chat_id}, %{id: object_id}) do
+    __MODULE__
+    |> Repo.get_by(chat_id: chat_id, object_id: object_id)
+    |> Repo.preload(:object)
+  end
+
+  def for_chat_query(chat) do
+    from(cr in __MODULE__,
+      where: cr.chat_id == ^chat.id,
+      order_by: [desc: :id],
+      preload: [:object]
+    )
+  end
+
+  def create(chat, object, seen) do
+    params = %{
+      chat_id: chat.id,
+      object_id: object.id,
+      seen: seen
+    }
+
+    %__MODULE__{}
+    |> changeset(params)
+    |> Repo.insert()
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index a34bf6a05..cda52b00e 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   """
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -104,6 +105,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
               Object.decrease_replies_count(in_reply_to)
             end
 
+            ChatMessageReference.delete_for_object(deleted_object)
+
             ActivityPub.stream_out(object)
             ActivityPub.stream_out_participations(deleted_object, user)
             :ok
@@ -137,9 +140,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
       |> Enum.each(fn [user, other_user] ->
         if user.local do
           if user.ap_id == actor.ap_id do
-            Chat.get_or_create(user.id, other_user.ap_id)
+            {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+            ChatMessageReference.create(chat, object, true)
           else
-            Chat.bump_or_create(user.id, other_user.ap_id)
+            {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+            ChatMessageReference.create(chat, object, false)
           end
         end
       end)
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 210c8ec4a..c54681054 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -6,14 +6,15 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Object
   alias Pleroma.Pagination
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
   alias Pleroma.Web.PleromaAPI.ChatView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
 
   import Ecto.Query
   import Pleroma.Web.ActivityPub.ObjectValidator, only: [stringify_keys: 1]
@@ -35,28 +36,38 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
 
-  def delete_message(%{assigns: %{user: %{ap_id: actor} = user}} = conn, %{
-        message_id: id
+  def delete_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
+        message_id: message_id,
+        id: chat_id
       }) do
-    with %Object{
-           data: %{
-             "actor" => ^actor,
-             "id" => object,
-             "to" => [recipient],
-             "type" => "ChatMessage"
-           }
-         } = message <- Object.get_by_id(id),
-         %Chat{} = chat <- Chat.get(user.id, recipient),
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id(object),
-         {:ok, _delete} <- CommonAPI.delete(activity.id, user) do
+    with %ChatMessageReference{} = cm_ref <-
+           ChatMessageReference.get_by_id(message_id),
+         ^chat_id <- cm_ref.chat_id |> to_string(),
+         %Chat{user_id: ^user_id} <- Chat.get_by_id(chat_id),
+         {:ok, _} <- remove_or_delete(cm_ref, user) do
       conn
-      |> put_view(ChatMessageView)
-      |> render("show.json", for: user, object: message, chat: chat)
+      |> put_view(ChatMessageReferenceView)
+      |> render("show.json", chat_message_reference: cm_ref)
     else
-      _e -> {:error, :could_not_delete}
+      _e ->
+        {:error, :could_not_delete}
     end
   end
 
+  defp remove_or_delete(
+         %{object: %{data: %{"actor" => actor, "id" => id}}},
+         %{ap_id: actor} = user
+       ) do
+    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
+      CommonAPI.delete(activity.id, user)
+    end
+  end
+
+  defp remove_or_delete(cm_ref, _) do
+    cm_ref
+    |> ChatMessageReference.delete()
+  end
+
   def post_chat_message(
         %{body_params: params, assigns: %{user: %{id: user_id} = user}} = conn,
         %{
@@ -69,10 +80,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
            CommonAPI.post_chat_message(user, recipient, params[:content],
              media_id: params[:media_id]
            ),
-         message <- Object.normalize(activity) do
+         message <- Object.normalize(activity, false),
+         cm_ref <- ChatMessageReference.for_chat_and_object(chat, message) do
       conn
-      |> put_view(ChatMessageView)
-      |> render("show.json", for: user, object: message, chat: chat)
+      |> put_view(ChatMessageReferenceView)
+      |> render("show.json", for: user, chat_message_reference: cm_ref)
     end
   end
 
@@ -87,14 +99,14 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
-      messages =
+      cm_refs =
         chat
-        |> Chat.messages_for_chat_query()
+        |> ChatMessageReference.for_chat_query()
         |> Pagination.fetch_paginated(params |> stringify_keys())
 
       conn
-      |> put_view(ChatMessageView)
-      |> render("index.json", for: user, objects: messages, chat: chat)
+      |> put_view(ChatMessageReferenceView)
+      |> render("index.json", for: user, chat_message_references: cm_refs)
     else
       _ ->
         conn
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
similarity index 67%
rename from lib/pleroma/web/pleroma_api/views/chat_message_view.ex
rename to lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
index b088a8734..ff170e162 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
@@ -2,10 +2,9 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
+defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceView do
   use Pleroma.Web, :view
 
-  alias Pleroma.Chat
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
@@ -13,8 +12,12 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
   def render(
         "show.json",
         %{
-          object: %{id: id, data: %{"type" => "ChatMessage"} = chat_message},
-          chat: %Chat{id: chat_id}
+          chat_message_reference: %{
+            id: id,
+            object: %{data: chat_message},
+            chat_id: chat_id,
+            seen: seen
+          }
         }
       ) do
     %{
@@ -26,11 +29,17 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
       emojis: StatusView.build_emojis(chat_message["emoji"]),
       attachment:
         chat_message["attachment"] &&
-          StatusView.render("attachment.json", attachment: chat_message["attachment"])
+          StatusView.render("attachment.json", attachment: chat_message["attachment"]),
+      seen: seen
     }
   end
 
   def render("index.json", opts) do
-    render_many(opts[:objects], __MODULE__, "show.json", Map.put(opts, :as, :object))
+    render_many(
+      opts[:chat_message_references],
+      __MODULE__,
+      "show.json",
+      Map.put(opts, :as, :chat_message_reference)
+    )
   end
 end
diff --git a/priv/repo/migrations/20200602150528_create_chat_message_reference.exs b/priv/repo/migrations/20200602150528_create_chat_message_reference.exs
new file mode 100644
index 000000000..6f9148b7c
--- /dev/null
+++ b/priv/repo/migrations/20200602150528_create_chat_message_reference.exs
@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.CreateChatMessageReference do
+  use Ecto.Migration
+
+  def change do
+    create table(:chat_message_references, primary_key: false) do
+      add(:id, :uuid, primary_key: true)
+      add(:chat_id, references(:chats, on_delete: :delete_all), null: false)
+      add(:object_id, references(:objects, on_delete: :delete_all), null: false)
+      add(:seen, :boolean, default: false, null: false)
+
+      timestamps()
+    end
+
+    create(index(:chat_message_references, [:chat_id, "id desc"]))
+  end
+end
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 210ba6ef0..ff6b3ac15 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -330,7 +331,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
       end
     end
 
-    test "it creates a Chat for the local users and bumps the unread count, except for the author" do
+    test "it creates a Chat and ChatMessageReferences for the local users and bumps the unread count, except for the author" do
       author = insert(:user, local: true)
       recipient = insert(:user, local: true)
 
@@ -347,8 +348,18 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
       chat = Chat.get(author.id, recipient.ap_id)
       assert chat.unread == 0
 
+      [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+
+      assert cm_ref.object.data["content"] == "hey"
+      assert cm_ref.seen == true
+
       chat = Chat.get(recipient.id, author.ap_id)
       assert chat.unread == 1
+
+      [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+
+      assert cm_ref.object.data["content"] == "hey"
+      assert cm_ref.seen == false
     end
 
     test "it creates a Chat for the local users and bumps the unread count" do
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index d79aa3148..bd4024c09 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -5,6 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -95,7 +96,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do
     setup do: oauth_access(["write:statuses"])
 
-    test "it deletes a message for the author of the message", %{conn: conn, user: user} do
+    test "it deletes a message from the chat", %{conn: conn, user: user} do
       recipient = insert(:user)
 
       {:ok, message} =
@@ -107,23 +108,32 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
       chat = Chat.get(user.id, recipient.ap_id)
 
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+
+      # Deleting your own message removes the message and the reference
       result =
         conn
         |> put_req_header("content-type", "application/json")
-        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{object.id}")
+        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{cm_ref.id}")
         |> json_response_and_validate_schema(200)
 
-      assert result["id"] == to_string(object.id)
+      assert result["id"] == cm_ref.id
+      refute ChatMessageReference.get_by_id(cm_ref.id)
+      assert %{data: %{"type" => "Tombstone"}} = Object.get_by_id(object.id)
 
+      # Deleting other people's messages just removes the reference
       object = Object.normalize(other_message, false)
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
       result =
         conn
         |> put_req_header("content-type", "application/json")
-        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{object.id}")
-        |> json_response(400)
+        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{cm_ref.id}")
+        |> json_response_and_validate_schema(200)
 
-      assert result == %{"error" => "could_not_delete"}
+      assert result["id"] == cm_ref.id
+      refute ChatMessageReference.get_by_id(cm_ref.id)
+      assert Object.get_by_id(object.id)
     end
   end
 
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_reference_view_test.exs
similarity index 67%
rename from test/web/pleroma_api/views/chat_message_view_test.exs
rename to test/web/pleroma_api/views/chat_message_reference_view_test.exs
index d7a2d10a5..00024d52c 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -2,14 +2,15 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
+defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
 
   import Pleroma.Factory
 
@@ -30,25 +31,32 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
 
     object = Object.normalize(activity)
 
-    chat_message = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-    assert chat_message[:id] == object.id |> to_string()
+    chat_message = ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
+
+    assert chat_message[:id] == cm_ref.id
     assert chat_message[:content] == "kippis :firefox:"
     assert chat_message[:account_id] == user.id
     assert chat_message[:chat_id]
     assert chat_message[:created_at]
+    assert chat_message[:seen] == true
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
 
     {:ok, activity} = CommonAPI.post_chat_message(recipient, user, "gkgkgk", media_id: upload.id)
 
     object = Object.normalize(activity)
 
-    chat_message_two = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-    assert chat_message_two[:id] == object.id |> to_string()
+    chat_message_two =
+      ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
+
+    assert chat_message_two[:id] == cm_ref.id
     assert chat_message_two[:content] == "gkgkgk"
     assert chat_message_two[:account_id] == recipient.id
     assert chat_message_two[:chat_id] == chat_message[:chat_id]
     assert chat_message_two[:attachment]
+    assert chat_message_two[:seen] == false
   end
 end