From 90d516d42bd3d29e71e364535dd4208f8a54992a Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Tue, 15 Oct 2019 16:52:41 +0200 Subject: [PATCH 01/39] Store status data inside flag activity --- lib/pleroma/web/activity_pub/utils.ex | 30 ++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 4ef479f96..57982eb4a 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -609,9 +609,33 @@ defmodule Pleroma.Web.ActivityPub.Utils do defp build_flag_object(%{account: account, statuses: statuses} = _) do [account.ap_id] ++ Enum.map(statuses || [], fn - %Activity{} = act -> act.data["id"] - act when is_map(act) -> act["id"] - act when is_binary(act) -> act + %Activity{} = act -> + obj = Object.get_by_ap_id(act.data["object"]) + + %{ + "type" => "Note", + "id" => act.data["id"], + "content" => obj.data["content"] + } + + act when is_map(act) -> + obj = Object.get_by_ap_id(act["object"]) + + %{ + "type" => "Note", + "id" => act["id"], + "content" => obj.data["content"] + } + + act + when is_binary(act) -> + activity = Activity.get_by_ap_id_with_object(act) + + %{ + "type" => "Note", + "id" => activity.data["id"], + "content" => activity.data["object"]["content"] + } end) end From b08b1d5d91968fbe94e20897ee3529216dd50a0a Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 23 Oct 2019 21:27:22 +0200 Subject: [PATCH 02/39] Store status data inside Flag activity --- lib/pleroma/web/activity_pub/utils.ex | 43 ++++++++----------- lib/pleroma/web/admin_api/report.ex | 4 +- test/web/activity_pub/activity_pub_test.exs | 13 +++++- test/web/activity_pub/transmogrifier_test.exs | 13 +++++- test/web/activity_pub/utils_test.exs | 11 ++++- test/web/common_api/common_api_test.exs | 11 ++++- 6 files changed, 62 insertions(+), 33 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 57982eb4a..c58ee7482 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.Endpoint alias Pleroma.Web.Router.Helpers @@ -608,34 +609,24 @@ defmodule Pleroma.Web.ActivityPub.Utils do defp build_flag_object(%{account: account, statuses: statuses} = _) do [account.ap_id] ++ - Enum.map(statuses || [], fn - %Activity{} = act -> - obj = Object.get_by_ap_id(act.data["object"]) + Enum.map(statuses || [], fn act -> + id = + case act do + %Activity{} = act -> act.data["id"] + act when is_map(act) -> act["id"] + act when is_binary(act) -> act + end - %{ - "type" => "Note", - "id" => act.data["id"], - "content" => obj.data["content"] - } + activity = Activity.get_by_ap_id_with_object(id) + actor = User.get_by_ap_id(activity.object.data["actor"]) - act when is_map(act) -> - obj = Object.get_by_ap_id(act["object"]) - - %{ - "type" => "Note", - "id" => act["id"], - "content" => obj.data["content"] - } - - act - when is_binary(act) -> - activity = Activity.get_by_ap_id_with_object(act) - - %{ - "type" => "Note", - "id" => activity.data["id"], - "content" => activity.data["object"]["content"] - } + %{ + "type" => "Note", + "id" => activity.data["id"], + "content" => activity.object.data["content"], + "published" => activity.object.data["published"], + "actor" => AccountView.render("show.json", %{user: actor}) + } end) end diff --git a/lib/pleroma/web/admin_api/report.ex b/lib/pleroma/web/admin_api/report.ex index c751dc2be..ccd56e15e 100644 --- a/lib/pleroma/web/admin_api/report.ex +++ b/lib/pleroma/web/admin_api/report.ex @@ -13,8 +13,8 @@ defmodule Pleroma.Web.AdminAPI.Report do account = User.get_cached_by_ap_id(account_ap_id) statuses = - Enum.map(status_ap_ids, fn ap_id -> - Activity.get_by_ap_id_with_object(ap_id) + Enum.map(status_ap_ids, fn act -> + Activity.get_by_ap_id_with_object(act["id"]) end) %{report: report, user: user, account: account, statuses: statuses} diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 3a5a2f984..cbd81b698 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -1265,6 +1266,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do target_ap_id = target_account.ap_id activity_ap_id = activity.data["id"] + activity_with_object = Activity.get_by_ap_id_with_object(activity_ap_id) + assert {:ok, activity} = ActivityPub.flag(%{ actor: reporter, @@ -1274,13 +1277,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do content: content }) + note_obj = %{ + "type" => "Note", + "id" => activity_ap_id, + "content" => content, + "published" => activity_with_object.object.data["published"], + "actor" => AccountView.render("show.json", %{user: target_account}) + } + assert %Activity{ actor: ^reporter_ap_id, data: %{ "type" => "Flag", "content" => ^content, "context" => ^context, - "object" => [^target_ap_id, ^activity_ap_id] + "object" => [^target_ap_id, ^note_obj] } } = activity end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6c35a6f4d..82389b344 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -12,6 +12,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus alias Pleroma.Web.Websub.WebsubClientSubscription @@ -1093,10 +1094,18 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) object = Object.normalize(activity) + note_obj = %{ + "type" => "Note", + "id" => activity.data["id"], + "content" => "test post", + "published" => object.data["published"], + "actor" => AccountView.render("show.json", %{user: user}) + } + message = %{ "@context" => "https://www.w3.org/ns/activitystreams", "cc" => [user.ap_id], - "object" => [user.ap_id, object.data["id"]], + "object" => [user.ap_id, activity], "type" => "Flag", "content" => "blocked AND reported!!!", "actor" => other_user.ap_id @@ -1104,7 +1113,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert {:ok, activity} = Transmogrifier.handle_incoming(message) - assert activity.data["object"] == [user.ap_id, object.data["id"]] + assert activity.data["object"] == [user.ap_id, note_obj] assert activity.data["content"] == "blocked AND reported!!!" assert activity.data["actor"] == other_user.ap_id assert activity.data["cc"] == [user.ap_id] diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index c57ea7eb9..4475d06d2 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -581,11 +582,19 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do %{} ) + note_obj = %{ + "type" => "Note", + "id" => activity_ap_id, + "content" => content, + "published" => activity.object.data["published"], + "actor" => AccountView.render("show.json", %{user: target_account}) + } + assert %{ "type" => "Flag", "content" => ^content, "context" => ^context, - "object" => [^target_ap_id, ^activity_ap_id], + "object" => [^target_ap_id, ^note_obj], "state" => "open" } = res end diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 83df44c36..709aa4dbe 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.CommonAPITest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -385,6 +386,14 @@ defmodule Pleroma.Web.CommonAPITest do "status_ids" => [activity.id] } + note_obj = %{ + "type" => "Note", + "id" => activity_ap_id, + "content" => "foobar", + "published" => activity.object.data["published"], + "actor" => AccountView.render("show.json", %{user: target_user}) + } + assert {:ok, flag_activity} = CommonAPI.report(reporter, report_data) assert %Activity{ @@ -392,7 +401,7 @@ defmodule Pleroma.Web.CommonAPITest do data: %{ "type" => "Flag", "content" => ^comment, - "object" => [^target_ap_id, ^activity_ap_id], + "object" => [^target_ap_id, ^note_obj], "state" => "open" } } = flag_activity From 8eff05d4c62c4d3300fee173cad84f75a0aafb4d Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 27 Oct 2019 16:05:32 +0300 Subject: [PATCH 03/39] Strip status data from Flag (when federating or closing/resolving report) --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 3 +- lib/pleroma/web/activity_pub/utils.ex | 23 ++++ test/web/activity_pub/activity_pub_test.exs | 122 +++++++++++++------ test/web/common_api/common_api_test.exs | 5 + 5 files changed, 119 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3ccfa4ea..4744567d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities - OStatus: Extract RSS functionality - Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) +- Store status data inside Flag activity ### Fixed - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1d34c4d7e..4cdf4876e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -491,7 +491,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with flag_data <- make_flag_data(params, additional), {:ok, activity} <- insert(flag_data, local), - :ok <- maybe_federate(activity) do + {:ok, stripped_activity} <- strip_report_status_data(activity), + :ok <- maybe_federate(stripped_activity) do Enum.each(User.all_superusers(), fn superuser -> superuser |> Pleroma.Emails.AdminEmail.report(actor, account, statuses, content) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c58ee7482..520cc1b0c 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -22,6 +22,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Pleroma.Constants @supported_object_types ["Article", "Note", "Video", "Page", "Question", "Answer", "Audio"] + @strip_status_report_states ~w(closed resolved) @supported_report_states ~w(open closed resolved) @valid_visibilities ~w(public unlisted private direct) @@ -673,6 +674,20 @@ defmodule Pleroma.Web.ActivityPub.Utils do #### Report-related helpers + def update_report_state(%Activity{} = activity, state) + when state in @strip_status_report_states do + {:ok, stripped_activity} = strip_report_status_data(activity) + + new_data = + activity.data + |> Map.put("state", state) + |> Map.put("object", stripped_activity.data["object"]) + + activity + |> Changeset.change(data: new_data) + |> Repo.update() + end + def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do new_data = Map.put(activity.data, "state", state) @@ -683,6 +698,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do def update_report_state(_, _), do: {:error, "Unsupported state"} + def strip_report_status_data(activity) do + [actor | reported_activities] = activity.data["object"] + stripped_activities = Enum.map(reported_activities, & &1["id"]) + new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) + + {:ok, %{activity | data: new_data}} + end + def update_activity_visibility(activity, visibility) when visibility in @valid_visibilities do [to, cc, recipients] = activity diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index cbd81b698..ddc6493c0 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1255,45 +1255,99 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert 3 = length(activities) end - test "it can create a Flag activity" do - reporter = insert(:user) - target_account = insert(:user) - {:ok, activity} = CommonAPI.post(target_account, %{"status" => "foobar"}) - context = Utils.generate_context_id() - content = "foobar" + describe "flag/1" do + setup do + reporter = insert(:user) + target_account = insert(:user) + content = "foobar" + {:ok, activity} = CommonAPI.post(target_account, %{"status" => content}) + context = Utils.generate_context_id() - reporter_ap_id = reporter.ap_id - target_ap_id = target_account.ap_id - activity_ap_id = activity.data["id"] + reporter_ap_id = reporter.ap_id + target_ap_id = target_account.ap_id + activity_ap_id = activity.data["id"] - activity_with_object = Activity.get_by_ap_id_with_object(activity_ap_id) + activity_with_object = Activity.get_by_ap_id_with_object(activity_ap_id) - assert {:ok, activity} = - ActivityPub.flag(%{ - actor: reporter, - context: context, - account: target_account, - statuses: [activity], - content: content - }) + {:ok, + %{ + reporter: reporter, + context: context, + target_account: target_account, + reported_activity: activity, + content: content, + activity_ap_id: activity_ap_id, + activity_with_object: activity_with_object, + reporter_ap_id: reporter_ap_id, + target_ap_id: target_ap_id + }} + end - note_obj = %{ - "type" => "Note", - "id" => activity_ap_id, - "content" => content, - "published" => activity_with_object.object.data["published"], - "actor" => AccountView.render("show.json", %{user: target_account}) - } + test "it can create a Flag activity", + %{ + reporter: reporter, + context: context, + target_account: target_account, + reported_activity: reported_activity, + content: content, + activity_ap_id: activity_ap_id, + activity_with_object: activity_with_object, + reporter_ap_id: reporter_ap_id, + target_ap_id: target_ap_id + } do + assert {:ok, activity} = + ActivityPub.flag(%{ + actor: reporter, + context: context, + account: target_account, + statuses: [reported_activity], + content: content + }) - assert %Activity{ - actor: ^reporter_ap_id, - data: %{ - "type" => "Flag", - "content" => ^content, - "context" => ^context, - "object" => [^target_ap_id, ^note_obj] - } - } = activity + note_obj = %{ + "type" => "Note", + "id" => activity_ap_id, + "content" => content, + "published" => activity_with_object.object.data["published"], + "actor" => AccountView.render("show.json", %{user: target_account}) + } + + assert %Activity{ + actor: ^reporter_ap_id, + data: %{ + "type" => "Flag", + "content" => ^content, + "context" => ^context, + "object" => [^target_ap_id, ^note_obj] + } + } = activity + end + + test_with_mock "strips status data from Flag, before federating it", + %{ + reporter: reporter, + context: context, + target_account: target_account, + reported_activity: reported_activity, + content: content + }, + Utils, + [:passthrough], + [] do + {:ok, activity} = + ActivityPub.flag(%{ + actor: reporter, + context: context, + account: target_account, + statuses: [reported_activity], + content: content + }) + + new_data = + put_in(activity.data, ["object"], [target_account.ap_id, reported_activity.data["id"]]) + + assert_called(Utils.maybe_federate(%{activity | data: new_data})) + end end test "fetch_activities/2 returns activities addressed to a list " do diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 709aa4dbe..d69ed38a7 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -421,6 +421,11 @@ defmodule Pleroma.Web.CommonAPITest do {:ok, report} = CommonAPI.update_report_state(report_id, "resolved") assert report.data["state"] == "resolved" + + [reported_user, activity_id] = report.data["object"] + + assert reported_user == target_user.ap_id + assert activity_id == activity.data["id"] end test "does not update report state when state is unsupported" do From 8282b6ac3d977e2e427c42ef3669d547170cc27d Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 27 Oct 2019 16:17:37 +0300 Subject: [PATCH 04/39] Make sure incoming flags are updated with status data --- test/web/activity_pub/transmogrifier_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 1069ade02..4df3024a6 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1102,7 +1102,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do message = %{ "@context" => "https://www.w3.org/ns/activitystreams", "cc" => [user.ap_id], - "object" => [user.ap_id, activity], + "object" => [user.ap_id, activity.data["id"]], "type" => "Flag", "content" => "blocked AND reported!!!", "actor" => other_user.ap_id From d56bc622755ea0a858bf086bc1f525c1752e4db8 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 27 Oct 2019 16:33:58 +0300 Subject: [PATCH 05/39] Fix report parsing --- lib/pleroma/web/admin_api/report.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/admin_api/report.ex b/lib/pleroma/web/admin_api/report.ex index ccd56e15e..9c3468570 100644 --- a/lib/pleroma/web/admin_api/report.ex +++ b/lib/pleroma/web/admin_api/report.ex @@ -13,8 +13,9 @@ defmodule Pleroma.Web.AdminAPI.Report do account = User.get_cached_by_ap_id(account_ap_id) statuses = - Enum.map(status_ap_ids, fn act -> - Activity.get_by_ap_id_with_object(act["id"]) + Enum.map(status_ap_ids, fn + act when is_map(act) -> Activity.get_by_ap_id_with_object(act["id"]) + act when is_binary(act) -> Activity.get_by_ap_id_with_object(act) end) %{report: report, user: user, account: account, statuses: statuses} From 0be9cb086b070858b041cd15ee149d1323952aab Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 27 Oct 2019 19:29:35 +0300 Subject: [PATCH 06/39] Add migration --- ...91027143434_add_defaults_to_all_tables.exs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs diff --git a/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs b/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs new file mode 100644 index 000000000..ab60f1313 --- /dev/null +++ b/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs @@ -0,0 +1,52 @@ +defmodule Pleroma.Repo.Migrations.AddDefaultsToAllTables do + use Ecto.Migration + + def up do + execute("ALTER TABLE activities + ALTER COLUMN recipients SET DEFAULT ARRAY[]::character varying[]") + + execute("ALTER TABLE filters + ALTER COLUMN whole_word SET DEFAULT true") + + execute("ALTER TABLE push_subscriptions + ALTER COLUMN data SET DEFAULT '{}'::jsonb") + + execute(~s(ALTER TABLE users + ALTER COLUMN following SET DEFAULT ARRAY[]::character varying[], + ALTER COLUMN tags SET DEFAULT ARRAY[]::character varying[], + ALTER COLUMN notification_settings SET DEFAULT + '{"followers": true, "follows": true, "non_follows": true, "non_followers": true}'::jsonb)) + + # irreversible updates + + execute( + "UPDATE activities SET recipients = ARRAY[]::character varying[] WHERE recipients IS NULL" + ) + + execute("UPDATE filters SET whole_word = true WHERE whole_word IS NULL") + + execute("UPDATE push_subscriptions SET data = '{}'::jsonb WHERE data IS NULL") + + execute("UPDATE users SET following = ARRAY[]::character varying[] WHERE following IS NULL") + execute("UPDATE users SET tags = ARRAY[]::character varying[] WHERE tags IS NULL") + execute(~s(UPDATE users SET notification_settings = + '{"followers": true, "follows": true, "non_follows": true, "non_followers": true}'::jsonb + WHERE notification_settings = '{}'::jsonb)) + end + + def down do + execute("ALTER TABLE activities + ALTER COLUMN recipients DROP DEFAULT") + + execute("ALTER TABLE filters + ALTER COLUMN whole_word DROP DEFAULT") + + execute("ALTER TABLE push_subscriptions + ALTER COLUMN data DROP DEFAULT") + + execute("ALTER TABLE users + ALTER COLUMN following DROP DEFAULT, + ALTER COLUMN tags DROP DEFAULT, + ALTER COLUMN notification_settings SET DEFAULT '{}'::jsonb") + end +end From 4b0893631f2d9f157c200f2787fe5154dc0b811e Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Mon, 28 Oct 2019 19:16:19 +0300 Subject: [PATCH 07/39] Complete defaults --- ...91027143434_add_defaults_to_all_tables.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs b/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs index ab60f1313..0d2794ad3 100644 --- a/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs +++ b/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs @@ -27,6 +27,25 @@ defmodule Pleroma.Repo.Migrations.AddDefaultsToAllTables do execute("UPDATE push_subscriptions SET data = '{}'::jsonb WHERE data IS NULL") + execute("UPDATE users SET source_data = '{}'::jsonb where source_data IS NULL") + execute("UPDATE users SET note_count = 0 where note_count IS NULL") + execute("UPDATE users SET background = '{}'::jsonb where background IS NULL") + execute("UPDATE users SET follower_count = 0 where follower_count IS NULL") + + execute( + "UPDATE users SET unread_conversation_count = 0 where unread_conversation_count IS NULL" + ) + + execute( + ~s(UPDATE users SET email_notifications = '{"digest": false}'::jsonb where email_notifications IS NULL) + ) + + execute("UPDATE users SET default_scope = 'public' where default_scope IS NULL") + + execute( + "UPDATE users SET pleroma_settings_store = '{}'::jsonb where pleroma_settings_store IS NULL" + ) + execute("UPDATE users SET following = ARRAY[]::character varying[] WHERE following IS NULL") execute("UPDATE users SET tags = ARRAY[]::character varying[] WHERE tags IS NULL") execute(~s(UPDATE users SET notification_settings = From 435d220700c694f1312bf213d0591054a309489a Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Mon, 28 Oct 2019 19:17:50 +0300 Subject: [PATCH 08/39] Move setDefaultsToTables to past to run before notNull migrations --- ...all_tables.exs => 20191025143434_add_defaults_to_tables.exs} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename priv/repo/migrations/{20191027143434_add_defaults_to_all_tables.exs => 20191025143434_add_defaults_to_tables.exs} (97%) diff --git a/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs b/priv/repo/migrations/20191025143434_add_defaults_to_tables.exs similarity index 97% rename from priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs rename to priv/repo/migrations/20191025143434_add_defaults_to_tables.exs index 0d2794ad3..d16ab19f9 100644 --- a/priv/repo/migrations/20191027143434_add_defaults_to_all_tables.exs +++ b/priv/repo/migrations/20191025143434_add_defaults_to_tables.exs @@ -1,4 +1,4 @@ -defmodule Pleroma.Repo.Migrations.AddDefaultsToAllTables do +defmodule Pleroma.Repo.Migrations.AddDefaultsToTables do use Ecto.Migration def up do From 56e42602b0037f92d86fcbbbafa798a323afc12d Mon Sep 17 00:00:00 2001 From: stwf Date: Mon, 28 Oct 2019 12:51:58 -0400 Subject: [PATCH 09/39] capture where errors are expected --- test/web/activity_pub/transmogrifier_test.exs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 2f25c40d2..7c3c4bb9b 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -746,7 +746,10 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Poison.decode!() |> Map.put("actor", ap_id) - assert :error == Transmogrifier.handle_incoming(data) + assert capture_log(fn -> + assert :error == Transmogrifier.handle_incoming(data) + end) =~ "Object containment failed" + assert User.get_cached_by_ap_id(ap_id) end @@ -1409,7 +1412,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "type" => "Announce" } - :error = Transmogrifier.handle_incoming(data) + assert capture_log(fn -> + :error = Transmogrifier.handle_incoming(data) + end) =~ "Object containment failed" end test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do @@ -1422,7 +1427,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "type" => "Announce" } - :error = Transmogrifier.handle_incoming(data) + assert capture_log(fn -> + :error = Transmogrifier.handle_incoming(data) + end) =~ "Object containment failed" end test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do @@ -1435,7 +1442,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "type" => "Announce" } - :error = Transmogrifier.handle_incoming(data) + assert capture_log(fn -> + :error = Transmogrifier.handle_incoming(data) + end) =~ "Object containment failed" end end @@ -1738,7 +1747,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do describe "get_obj_helper/2" do test "returns nil when cannot normalize object" do - refute Transmogrifier.get_obj_helper("test-obj-id") + assert capture_log(fn -> + refute Transmogrifier.get_obj_helper("test-obj-id") + end) =~ "Unsupported URI scheme" end test "returns {:ok, %Object{}} for success case" do From 05d111b622c7496260effd298163e9ca2aa59c3d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 01:15:28 +0300 Subject: [PATCH 10/39] Use the default scrubber before mediaproxy transformation --- config/config.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index a69d41d17..e2b00bda1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -284,8 +284,8 @@ config :pleroma, :markup, allow_tables: false, allow_fonts: false, scrub_policy: [ - Pleroma.HTML.Transform.MediaProxy, - Pleroma.HTML.Scrubber.Default + Pleroma.HTML.Scrubber.Default, + Pleroma.HTML.Transform.MediaProxy ] config :pleroma, :frontend_configurations, From 9434e151a9f08067bf324ef027d857b67b0a96cb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 01:16:22 +0300 Subject: [PATCH 11/39] Blacklist myhtmlex nodes from swarm --- config/config.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.exs b/config/config.exs index e2b00bda1..81d50cdee 100644 --- a/config/config.exs +++ b/config/config.exs @@ -603,6 +603,7 @@ config :pleroma, :web_cache_ttl, activity_pub: nil, activity_pub_question: 30_000 +config :swarm, node_blacklist: [~r/myhtmlex_.*$/] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" From 2453928b57d3dbe60a5f6a4a85f2f03d3192108e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 01:16:57 +0300 Subject: [PATCH 12/39] Ensure myhtmlex is started before swarm Otherwise swarm is going to attempt resizing the cluster and timeout --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index dcb9d9ea8..705d56f0e 100644 --- a/mix.exs +++ b/mix.exs @@ -63,7 +63,7 @@ defmodule Pleroma.Mixfile do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :quack], + extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :myhtmlex, :swarm], included_applications: [:ex_syslogger] ] end From 08f68370659597d6bc428e425925bcb9516d5706 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 01:18:08 +0300 Subject: [PATCH 13/39] Switch from HtmlSanitizeEx to FastSanitize --- lib/pleroma/html.ex | 129 +++++++++--------- test/emoji/formatter_test.exs | 2 +- test/html_test.exs | 44 +++--- .../mrf/normalize_markup_test.exs | 10 +- test/web/common_api/common_api_test.exs | 4 +- 5 files changed, 95 insertions(+), 94 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 937bafed5..fd0495049 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTML do - alias HtmlSanitizeEx.Scrubber defp get_scrubbers(scrubber) when is_atom(scrubber), do: [scrubber] defp get_scrubbers(scrubbers) when is_list(scrubbers), do: scrubbers @@ -24,9 +23,13 @@ defmodule Pleroma.HTML do end) end - def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber) + def filter_tags(html, scrubber) do + {:ok, content} = FastSanitize.Sanitizer.scrub(html, scrubber) + content + end + def filter_tags(html), do: filter_tags(html, nil) - def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags) + def strip_tags(html), do: filter_tags(html, FastSanitize.Sanitizer.StripTags) def get_cached_scrubbed_html_for_activity( content, @@ -36,7 +39,6 @@ defmodule Pleroma.HTML do callback \\ fn x -> x end ) do key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}" - Cachex.fetch!(:scrubber_cache, key, fn _key -> object = Pleroma.Object.normalize(activity) ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback) @@ -46,7 +48,7 @@ defmodule Pleroma.HTML do def get_cached_stripped_html_for_activity(content, activity, key) do get_cached_scrubbed_html_for_activity( content, - HtmlSanitizeEx.Scrubber.StripTags, + FastSanitize.Sanitizer.StripTags, activity, key, &HtmlEntities.decode/1 @@ -109,13 +111,12 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do require HtmlSanitizeEx.Scrubber.Meta alias HtmlSanitizeEx.Scrubber.Meta - Meta.remove_cdata_sections_before_scrub() Meta.strip_comments() # links - Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes) + Meta.allow_tag_with_uri_attributes(:a, ["href", "data-user", "data-tag"], @valid_schemes) - Meta.allow_tag_with_this_attribute_values("a", "class", [ + Meta.allow_tag_with_this_attribute_values(:a, "class", [ "hashtag", "u-url", "mention", @@ -123,29 +124,29 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do "mention u-url" ]) - Meta.allow_tag_with_this_attribute_values("a", "rel", [ + Meta.allow_tag_with_this_attribute_values(:a, "rel", [ "tag", "nofollow", "noopener", "noreferrer" ]) - Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.allow_tag_with_these_attributes(:a, ["name", "title"]) # paragraphs and linebreaks - Meta.allow_tag_with_these_attributes("br", []) - Meta.allow_tag_with_these_attributes("p", []) + Meta.allow_tag_with_these_attributes(:br, []) + Meta.allow_tag_with_these_attributes(:p, []) # microformats - Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"]) - Meta.allow_tag_with_these_attributes("span", []) + Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card"]) + Meta.allow_tag_with_these_attributes(:span, []) # allow inline images for custom emoji if Pleroma.Config.get([:markup, :allow_inline_images]) do # restrict img tags to http/https only, because of MediaProxy. - Meta.allow_tag_with_uri_attributes("img", ["src"], ["http", "https"]) + Meta.allow_tag_with_uri_attributes(:img, ["src"], ["http", "https"]) - Meta.allow_tag_with_these_attributes("img", [ + Meta.allow_tag_with_these_attributes(:img, [ "width", "height", "class", @@ -160,19 +161,19 @@ end defmodule Pleroma.HTML.Scrubber.Default do @doc "The default HTML scrubbing policy: no " - require HtmlSanitizeEx.Scrubber.Meta - alias HtmlSanitizeEx.Scrubber.Meta + require FastSanitize.Sanitizer.Meta + alias FastSanitize.Sanitizer.Meta # credo:disable-for-previous-line # No idea how to fix this one… @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) - Meta.remove_cdata_sections_before_scrub() +# Meta.remove_cdata_sections_before_scrub() Meta.strip_comments() - Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes) + Meta.allow_tag_with_uri_attributes(:a, ["href", "data-user", "data-tag"], @valid_schemes) - Meta.allow_tag_with_this_attribute_values("a", "class", [ + Meta.allow_tag_with_this_attribute_values(:a, "class", [ "hashtag", "u-url", "mention", @@ -180,7 +181,7 @@ defmodule Pleroma.HTML.Scrubber.Default do "mention u-url" ]) - Meta.allow_tag_with_this_attribute_values("a", "rel", [ + Meta.allow_tag_with_this_attribute_values(:a, "rel", [ "tag", "nofollow", "noopener", @@ -188,37 +189,37 @@ defmodule Pleroma.HTML.Scrubber.Default do "ugc" ]) - Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.allow_tag_with_these_attributes(:a, ["name", "title"]) - Meta.allow_tag_with_these_attributes("abbr", ["title"]) + Meta.allow_tag_with_these_attributes(:abbr, ["title"]) - Meta.allow_tag_with_these_attributes("b", []) - Meta.allow_tag_with_these_attributes("blockquote", []) - Meta.allow_tag_with_these_attributes("br", []) - Meta.allow_tag_with_these_attributes("code", []) - Meta.allow_tag_with_these_attributes("del", []) - Meta.allow_tag_with_these_attributes("em", []) - Meta.allow_tag_with_these_attributes("i", []) - Meta.allow_tag_with_these_attributes("li", []) - Meta.allow_tag_with_these_attributes("ol", []) - Meta.allow_tag_with_these_attributes("p", []) - Meta.allow_tag_with_these_attributes("pre", []) - Meta.allow_tag_with_these_attributes("strong", []) - Meta.allow_tag_with_these_attributes("sub", []) - Meta.allow_tag_with_these_attributes("sup", []) - Meta.allow_tag_with_these_attributes("u", []) - Meta.allow_tag_with_these_attributes("ul", []) + Meta.allow_tag_with_these_attributes(:b, []) + Meta.allow_tag_with_these_attributes(:blockquote, []) + Meta.allow_tag_with_these_attributes(:br, []) + Meta.allow_tag_with_these_attributes(:code, []) + Meta.allow_tag_with_these_attributes(:del, []) + Meta.allow_tag_with_these_attributes(:em, []) + Meta.allow_tag_with_these_attributes(:i, []) + Meta.allow_tag_with_these_attributes(:li, []) + Meta.allow_tag_with_these_attributes(:ol, []) + Meta.allow_tag_with_these_attributes(:p, []) + Meta.allow_tag_with_these_attributes(:pre, []) + Meta.allow_tag_with_these_attributes(:strong, []) + Meta.allow_tag_with_these_attributes(:sub, []) + Meta.allow_tag_with_these_attributes(:sup, []) + Meta.allow_tag_with_these_attributes(:u, []) + Meta.allow_tag_with_these_attributes(:ul, []) - Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"]) - Meta.allow_tag_with_these_attributes("span", []) + Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card"]) + Meta.allow_tag_with_these_attributes(:span, []) @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images]) if @allow_inline_images do # restrict img tags to http/https only, because of MediaProxy. - Meta.allow_tag_with_uri_attributes("img", ["src"], ["http", "https"]) + Meta.allow_tag_with_uri_attributes(:img, ["src"], ["http", "https"]) - Meta.allow_tag_with_these_attributes("img", [ + Meta.allow_tag_with_these_attributes(:img, [ "width", "height", "class", @@ -228,24 +229,24 @@ defmodule Pleroma.HTML.Scrubber.Default do end if Pleroma.Config.get([:markup, :allow_tables]) do - Meta.allow_tag_with_these_attributes("table", []) - Meta.allow_tag_with_these_attributes("tbody", []) - Meta.allow_tag_with_these_attributes("td", []) - Meta.allow_tag_with_these_attributes("th", []) - Meta.allow_tag_with_these_attributes("thead", []) - Meta.allow_tag_with_these_attributes("tr", []) + Meta.allow_tag_with_these_attributes(:table, []) + Meta.allow_tag_with_these_attributes(:tbody, []) + Meta.allow_tag_with_these_attributes(:td, []) + Meta.allow_tag_with_these_attributes(:th, []) + Meta.allow_tag_with_these_attributes(:thead, []) + Meta.allow_tag_with_these_attributes(:tr, []) end if Pleroma.Config.get([:markup, :allow_headings]) do - Meta.allow_tag_with_these_attributes("h1", []) - Meta.allow_tag_with_these_attributes("h2", []) - Meta.allow_tag_with_these_attributes("h3", []) - Meta.allow_tag_with_these_attributes("h4", []) - Meta.allow_tag_with_these_attributes("h5", []) + Meta.allow_tag_with_these_attributes(:h1, []) + Meta.allow_tag_with_these_attributes(:h2, []) + Meta.allow_tag_with_these_attributes(:h3, []) + Meta.allow_tag_with_these_attributes(:h4, []) + Meta.allow_tag_with_these_attributes(:h5, []) end if Pleroma.Config.get([:markup, :allow_fonts]) do - Meta.allow_tag_with_these_attributes("font", ["face"]) + Meta.allow_tag_with_these_attributes(:font, ["face"]) end Meta.strip_everything_not_covered() @@ -258,7 +259,7 @@ defmodule Pleroma.HTML.Transform.MediaProxy do def before_scrub(html), do: html - def scrub_attribute("img", {"src", "http" <> target}) do + def scrub_attribute(:img, {"src", "http" <> target}) do media_url = ("http" <> target) |> MediaProxy.url() @@ -268,16 +269,16 @@ defmodule Pleroma.HTML.Transform.MediaProxy do def scrub_attribute(_tag, attribute), do: attribute - def scrub({"img", attributes, children}) do + def scrub({:img, attributes, children}) do attributes = attributes - |> Enum.map(fn attr -> scrub_attribute("img", attr) end) + |> Enum.map(fn attr -> scrub_attribute(:img, attr) end) |> Enum.reject(&is_nil(&1)) - {"img", attributes, children} + {:img, attributes, children} end - def scrub({:comment, _children}), do: "" + def scrub({:comment, _text, _children}), do: "" def scrub({tag, attributes, children}), do: {tag, attributes, children} def scrub({_tag, children}), do: children @@ -298,9 +299,9 @@ defmodule Pleroma.HTML.Scrubber.LinksOnly do Meta.strip_comments() # links - Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes) + Meta.allow_tag_with_uri_attributes(:a, ["href"], @valid_schemes) - Meta.allow_tag_with_this_attribute_values("a", "rel", [ + Meta.allow_tag_with_this_attribute_values(:a, "rel", [ "tag", "nofollow", "noopener", @@ -309,6 +310,6 @@ defmodule Pleroma.HTML.Scrubber.LinksOnly do "ugc" ]) - Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.allow_tag_with_these_attributes(:a, ["name", "title"]) Meta.strip_everything_not_covered() end diff --git a/test/emoji/formatter_test.exs b/test/emoji/formatter_test.exs index 6d25fc453..3e37ae3f0 100644 --- a/test/emoji/formatter_test.exs +++ b/test/emoji/formatter_test.exs @@ -12,7 +12,7 @@ defmodule Pleroma.Emoji.FormatterTest do text = "I love :firefox:" expected_result = - "I love \"firefox\"" + "I love \"firefox\"" assert Formatter.emojify(text) == expected_result end diff --git a/test/html_test.exs b/test/html_test.exs index 306ad3b3b..f0869534c 100644 --- a/test/html_test.exs +++ b/test/html_test.exs @@ -21,31 +21,31 @@ defmodule Pleroma.HTMLTest do """ @html_onerror_sample """ - + """ @html_span_class_sample """ - hi + hi """ @html_span_microformats_sample """ - @foo + @foo """ @html_span_invalid_microformats_sample """ - @foo + @foo """ describe "StripTags scrubber" do test "works as expected" do expected = """ - this is in bold + this is in bold this is a paragraph this is a linebreak - this is a link with allowed "rel" attribute: example.com - this is a link with not allowed "rel" attribute: example.com + this is a link with allowed "rel" attribute: example.com + this is a link with not allowed "rel" attribute: example.com this is an image: - alert('hacked') + alert('hacked') """ assert expected == HTML.strip_tags(@html_sample) @@ -61,13 +61,13 @@ defmodule Pleroma.HTMLTest do describe "TwitterText scrubber" do test "normalizes HTML as expected" do expected = """ - this is in bold + this is in bold

this is a paragraph

- this is a linebreak
- this is a link with allowed "rel" attribute: - this is a link with not allowed "rel" attribute: example.com - this is an image:
- alert('hacked') + this is a linebreak
+ this is a link with allowed "rel" attribute: + this is a link with not allowed "rel" attribute: example.com + this is an image:
+ alert('hacked') """ assert expected == HTML.filter_tags(@html_sample, Pleroma.HTML.Scrubber.TwitterText) @@ -75,7 +75,7 @@ defmodule Pleroma.HTMLTest do test "does not allow attribute-based XSS" do expected = """ - + """ assert expected == HTML.filter_tags(@html_onerror_sample, Pleroma.HTML.Scrubber.TwitterText) @@ -115,13 +115,13 @@ defmodule Pleroma.HTMLTest do describe "default scrubber" do test "normalizes HTML as expected" do expected = """ - this is in bold + this is in bold

this is a paragraph

- this is a linebreak
- this is a link with allowed "rel" attribute: - this is a link with not allowed "rel" attribute: example.com - this is an image:
- alert('hacked') + this is a linebreak
+ this is a link with allowed "rel" attribute: + this is a link with not allowed "rel" attribute: example.com + this is an image:
+ alert('hacked') """ assert expected == HTML.filter_tags(@html_sample, Pleroma.HTML.Scrubber.Default) @@ -129,7 +129,7 @@ defmodule Pleroma.HTMLTest do test "does not allow attribute-based XSS" do expected = """ - + """ assert expected == HTML.filter_tags(@html_onerror_sample, Pleroma.HTML.Scrubber.Default) diff --git a/test/web/activity_pub/mrf/normalize_markup_test.exs b/test/web/activity_pub/mrf/normalize_markup_test.exs index 3916a1f35..0207be56b 100644 --- a/test/web/activity_pub/mrf/normalize_markup_test.exs +++ b/test/web/activity_pub/mrf/normalize_markup_test.exs @@ -20,11 +20,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do expected = """ this is in bold

this is a paragraph

- this is a linebreak
- this is a link with allowed "rel" attribute: - this is a link with not allowed "rel" attribute: example.com - this is an image:
- alert('hacked') + this is a linebreak
+ this is a link with allowed "rel" attribute: + this is a link with not allowed "rel" attribute: example.com + this is an image:
+ alert('hacked') """ message = %{"type" => "Create", "object" => %{"content" => @html_sample}} diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 1d2f20617..212b00cbb 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -140,7 +140,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity) - assert object.data["content"] == "

2hu

alert('xss')" + assert object.data["content"] == "

2hu

alert('xss')" end test "it filters out obviously bad tags when accepting a post as Markdown" do @@ -156,7 +156,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity) - assert object.data["content"] == "

2hu

alert('xss')" + assert object.data["content"] == "

2hu

alert('xss')" end test "it does not allow replies to direct messages that are not direct messages themselves" do From 9cde3321a3b4ac6a8df16d7e59f169b545afe55f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 01:21:00 +0300 Subject: [PATCH 14/39] Add fast_sanitize to dependencies --- mix.exs | 1 + mix.lock | 49 ++++++++++++++++++++++++++----------------------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/mix.exs b/mix.exs index 705d56f0e..0e7c6a6f8 100644 --- a/mix.exs +++ b/mix.exs @@ -109,6 +109,7 @@ defmodule Pleroma.Mixfile do {:pbkdf2_elixir, "~> 0.12.3"}, {:trailing_format_plug, "~> 0.0.7"}, {:html_sanitize_ex, "~> 1.3.0"}, + {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "2820d841393b3c2f2183d0377f572bf00f513b7c"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, diff --git a/mix.lock b/mix.lock index 5f740638c..18e2572dd 100644 --- a/mix.lock +++ b/mix.lock @@ -13,44 +13,45 @@ "comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm"}, "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, + "cowboy": {:hex, :cowboy, "2.7.0", "91ed100138a764355f43316b1d23d7ff6bdb0de4ea618cb5d8677c93a7a2f115", [:rebar3], [{:cowlib, "~> 2.8.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, + "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, - "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, + "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm"}, "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, - "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.2.0", "940e2598813f205223d60c78d66e514afe1db5167ed8075510a59e496619cfb5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.3", "51274df79862845b388733fddcf6f107d0c8c86e27abe7131fa98f8d30761bda", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, - "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, + "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, - "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, - "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, + "ex_aws": {:hex, :ex_aws, "2.1.1", "1e4de2106cfbf4e837de41be41cd15813eabc722315e388f0d6bb3732cec47cd", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.2", "c0258bbdfea55de4f98f0b2f0ca61fe402cc696f573815134beb1866e778f47b", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, - "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "2820d841393b3c2f2183d0377f572bf00f513b7c", [ref: "2820d841393b3c2f2183d0377f572bf00f513b7c"]}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, - "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, - "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"}, + "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, + "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, - "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, + "gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]}, - "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "httpoison": {:hex, :httpoison, "1.6.1", "2ce5bf6e535cd0ab02e905ba8c276580bab80052c5c549f53ddea52d72e81f33", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, - "joken": {:hex, :joken, "2.0.1", "ec9ab31bf660f343380da033b3316855197c8d4c6ef597fa3fcb451b326beb14", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm"}, + "joken": {:hex, :joken, "2.1.0", "bf21a73105d82649f617c5e59a7f8919aa47013d2519ebcc39d998d8d12adda9", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm"}, "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, @@ -63,16 +64,18 @@ "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, + "myhtmlex": {:git, "https://github.com/rinpatch/myhtmlex.git", "d973dfb1b252b1c6e6eddddc18c0895aa977091c", [ref: "d973dfb1b252b1c6e6eddddc18c0895aa977091c", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, + "nodex": {:git, "https://github.com/rinpatch/nodex", "12ca7a2c5b5791f1e847d73ed646cf006d4c8ca8", [ref: "12ca7a2c5b5791f1e847d73ed646cf006d4c8ca8"]}, "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, - "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, - "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, + "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, + "phoenix_html": {:hex, :phoenix_html, "2.13.3", "850e292ff6e204257f5f9c4c54a8cb1f6fbc16ed53d360c2b780a3d0ba333867", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm"}, "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.2.0", "a7e0b32077cd6d2323ae15198839b05d9caddfa20663fd85787479e81f89520e", [:mix], [{:phoenix, "~> 1.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 0.1", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm"}, - "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, + "plug": {:hex, :plug, "1.8.3", "12d5f9796dc72e8ac9614e94bda5e51c4c028d0d428e9297650d09e15a684478", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "plug_cowboy": {:hex, :plug_cowboy, "2.1.0", "b75768153c3a8a9e8039d4b25bb9b14efbc58e9c4a6e6a270abff1cd30cbe320", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, @@ -80,7 +83,7 @@ "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.15.1", "23ce3417de70f4c0e9e7419ad85bdabcc6860a6925fe2c6f3b1b5b1e8e47bf2f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, - "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, @@ -92,16 +95,16 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, - "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, + "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, - "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, + "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "ueberauth": {:hex, :ueberauth, "0.6.2", "25a31111249d60bad8b65438b2306a4dc91f3208faa62f5a8c33e8713989b2e8", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm"}, - "web_push_encryption": {:hex, :web_push_encryption, "0.2.1", "d42cecf73420d9dc0053ba3299cc8c8d6ff2be2487d67ca2a57265868e4d9a98", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, + "web_push_encryption": {:hex, :web_push_encryption, "0.2.3", "a0ceab85a805a30852f143d22d71c434046fbdbafbc7292e7887cec500826a80", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, } From 41c23251f16ea1951cfea7da3fc809b9e71c25e8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 13:05:20 +0300 Subject: [PATCH 15/39] Bump fast_sanitize version --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 0e7c6a6f8..faff7033c 100644 --- a/mix.exs +++ b/mix.exs @@ -109,7 +109,7 @@ defmodule Pleroma.Mixfile do {:pbkdf2_elixir, "~> 0.12.3"}, {:trailing_format_plug, "~> 0.0.7"}, {:html_sanitize_ex, "~> 1.3.0"}, - {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "2820d841393b3c2f2183d0377f572bf00f513b7c"}, + {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "0d996cac23e86afc342a24047e77443661bbbb13"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, From 3e09b7c5ae2d8e6d3abb2020f2c63caad278e73f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 16:56:24 +0300 Subject: [PATCH 16/39] Fix two migrations sharing the same module name This makes ecto execute only the latter one. --- .../20191029101340_migrate_missing_follow_requests.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20191029101340_migrate_missing_follow_requests.exs b/priv/repo/migrations/20191029101340_migrate_missing_follow_requests.exs index 1b2666f3a..90b18efc8 100644 --- a/priv/repo/migrations/20191029101340_migrate_missing_follow_requests.exs +++ b/priv/repo/migrations/20191029101340_migrate_missing_follow_requests.exs @@ -1,4 +1,4 @@ -defmodule Pleroma.Repo.Migrations.MigrateFollowingRelationships do +defmodule Pleroma.Repo.Migrations.MigrateMissingFollowingRelationships do use Ecto.Migration def change do From 7358cf1f2041329f0924fc60106dfb5ec0288feb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 17:00:03 +0300 Subject: [PATCH 17/39] Bump fast_sanitize dependency --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index faff7033c..e46f9b2a4 100644 --- a/mix.exs +++ b/mix.exs @@ -109,7 +109,7 @@ defmodule Pleroma.Mixfile do {:pbkdf2_elixir, "~> 0.12.3"}, {:trailing_format_plug, "~> 0.0.7"}, {:html_sanitize_ex, "~> 1.3.0"}, - {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "0d996cac23e86afc342a24047e77443661bbbb13"}, + {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "a23fbf1bd77b002a8f72a53c39c53b176f98f81a"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, diff --git a/mix.lock b/mix.lock index 18e2572dd..01f6860a6 100644 --- a/mix.lock +++ b/mix.lock @@ -36,7 +36,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "2820d841393b3c2f2183d0377f572bf00f513b7c", [ref: "2820d841393b3c2f2183d0377f572bf00f513b7c"]}, + "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "a23fbf1bd77b002a8f72a53c39c53b176f98f81a", [ref: "a23fbf1bd77b002a8f72a53c39c53b176f98f81a"]}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, @@ -64,7 +64,7 @@ "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, - "myhtmlex": {:git, "https://github.com/rinpatch/myhtmlex.git", "d973dfb1b252b1c6e6eddddc18c0895aa977091c", [ref: "d973dfb1b252b1c6e6eddddc18c0895aa977091c", submodules: true]}, + "myhtmlex": {:git, "https://github.com/rinpatch/myhtmlex.git", "2031aeb1098e128f35a6a0b54841e259fc7672d5", [ref: "2031aeb1098e128f35a6a0b54841e259fc7672d5", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "nodex": {:git, "https://github.com/rinpatch/nodex", "12ca7a2c5b5791f1e847d73ed646cf006d4c8ca8", [ref: "12ca7a2c5b5791f1e847d73ed646cf006d4c8ca8"]}, "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, From 61400d7eb785f41ab853f42d0a221357253c39eb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 20:49:32 +0300 Subject: [PATCH 18/39] Fix XSS emoji test --- mix.exs | 2 +- mix.lock | 6 +++--- test/emoji/formatter_test.exs | 5 +---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mix.exs b/mix.exs index e46f9b2a4..2e64b79ad 100644 --- a/mix.exs +++ b/mix.exs @@ -109,7 +109,7 @@ defmodule Pleroma.Mixfile do {:pbkdf2_elixir, "~> 0.12.3"}, {:trailing_format_plug, "~> 0.0.7"}, {:html_sanitize_ex, "~> 1.3.0"}, - {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "a23fbf1bd77b002a8f72a53c39c53b176f98f81a"}, + {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, diff --git a/mix.lock b/mix.lock index 01f6860a6..6cbe54a76 100644 --- a/mix.lock +++ b/mix.lock @@ -36,7 +36,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "a23fbf1bd77b002a8f72a53c39c53b176f98f81a", [ref: "a23fbf1bd77b002a8f72a53c39c53b176f98f81a"]}, + "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c", [ref: "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c"]}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, @@ -64,9 +64,9 @@ "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, - "myhtmlex": {:git, "https://github.com/rinpatch/myhtmlex.git", "2031aeb1098e128f35a6a0b54841e259fc7672d5", [ref: "2031aeb1098e128f35a6a0b54841e259fc7672d5", submodules: true]}, + "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "eafbba443260c0055ef2da3918b351b112ff356f", [ref: "eafbba443260c0055ef2da3918b351b112ff356f", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, - "nodex": {:git, "https://github.com/rinpatch/nodex", "12ca7a2c5b5791f1e847d73ed646cf006d4c8ca8", [ref: "12ca7a2c5b5791f1e847d73ed646cf006d4c8ca8"]}, + "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "2927091d96900fb76f6bc897e46a6abb9070ebbd", [ref: "2927091d96900fb76f6bc897e46a6abb9070ebbd"]}, "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, diff --git a/test/emoji/formatter_test.exs b/test/emoji/formatter_test.exs index 3e37ae3f0..fda80d470 100644 --- a/test/emoji/formatter_test.exs +++ b/test/emoji/formatter_test.exs @@ -28,10 +28,7 @@ defmodule Pleroma.Emoji.FormatterTest do } |> Pleroma.Emoji.build() - expected_result = - "I love \"\"" - - assert Formatter.emojify(text, [{custom_emoji.code, custom_emoji}]) == expected_result + refute Formatter.emojify(text, [{custom_emoji.code, custom_emoji}]) =~ text end end From 77cfb08b8c4c07406af8b338ce010307f6af75cb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 29 Oct 2019 20:58:54 +0300 Subject: [PATCH 19/39] Remove commented-out code --- lib/pleroma/html.ex | 3 +-- mix.exs | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index fd0495049..294bc75f9 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTML do - defp get_scrubbers(scrubber) when is_atom(scrubber), do: [scrubber] defp get_scrubbers(scrubbers) when is_list(scrubbers), do: scrubbers defp get_scrubbers(_), do: [Pleroma.HTML.Scrubber.Default] @@ -39,6 +38,7 @@ defmodule Pleroma.HTML do callback \\ fn x -> x end ) do key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}" + Cachex.fetch!(:scrubber_cache, key, fn _key -> object = Pleroma.Object.normalize(activity) ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback) @@ -168,7 +168,6 @@ defmodule Pleroma.HTML.Scrubber.Default do @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) -# Meta.remove_cdata_sections_before_scrub() Meta.strip_comments() Meta.allow_tag_with_uri_attributes(:a, ["href", "data-user", "data-tag"], @valid_schemes) diff --git a/mix.exs b/mix.exs index 2e64b79ad..91b79dde2 100644 --- a/mix.exs +++ b/mix.exs @@ -109,7 +109,9 @@ defmodule Pleroma.Mixfile do {:pbkdf2_elixir, "~> 0.12.3"}, {:trailing_format_plug, "~> 0.0.7"}, {:html_sanitize_ex, "~> 1.3.0"}, - {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c"}, + {:fast_sanitize, + git: "https://git.pleroma.social/pleroma/fast_sanitize.git", + ref: "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, From ae59b38203b5358ddbf7f2cc5e2cbc816d171452 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 30 Oct 2019 09:20:13 +0300 Subject: [PATCH 20/39] Rip out the rest of htmlsanitizeex --- lib/pleroma/bbs/handler.ex | 3 ++- lib/pleroma/html.ex | 9 ++++----- mix.exs | 1 - test/web/mastodon_api/views/status_view_test.exs | 9 +++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index fa838a4e4..386afee89 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -5,6 +5,7 @@ defmodule Pleroma.BBS.Handler do use Sshd.ShellHandler alias Pleroma.Activity + alias Pleroma.HTML alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -44,7 +45,7 @@ defmodule Pleroma.BBS.Handler do def puts_activity(activity) do status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity}) IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})") - IO.puts(HtmlSanitizeEx.strip_tags(status.content)) + IO.puts(HTML.strip_tags(status.content)) IO.puts("") end diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 294bc75f9..997e965f0 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -108,8 +108,8 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) - require HtmlSanitizeEx.Scrubber.Meta - alias HtmlSanitizeEx.Scrubber.Meta + require FastSanitize.Sanitizer.Meta + alias FastSanitize.Sanitizer.Meta Meta.strip_comments() @@ -291,10 +291,9 @@ defmodule Pleroma.HTML.Scrubber.LinksOnly do @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) - require HtmlSanitizeEx.Scrubber.Meta - alias HtmlSanitizeEx.Scrubber.Meta + require FastSanitize.Sanitizer.Meta + alias FastSanitize.Sanitizer.Meta - Meta.remove_cdata_sections_before_scrub() Meta.strip_comments() # links diff --git a/mix.exs b/mix.exs index 91b79dde2..e85c2aa50 100644 --- a/mix.exs +++ b/mix.exs @@ -108,7 +108,6 @@ defmodule Pleroma.Mixfile do {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, {:trailing_format_plug, "~> 0.0.7"}, - {:html_sanitize_ex, "~> 1.3.0"}, {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", ref: "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c"}, diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index c200ad8fe..7526f2f27 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -107,7 +108,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do in_reply_to_account_id: nil, card: nil, reblog: nil, - content: HtmlSanitizeEx.basic_html(object_data["content"]), + content: HTML.filter_tags(object_data["content"]), created_at: created_at, reblogs_count: 0, replies_count: 0, @@ -119,7 +120,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do pinned: false, sensitive: false, poll: nil, - spoiler_text: HtmlSanitizeEx.basic_html(object_data["summary"]), + spoiler_text: HTML.filter_tags(object_data["summary"]), visibility: "public", media_attachments: [], mentions: [], @@ -146,8 +147,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do local: true, conversation_id: convo_id, in_reply_to_account_acct: nil, - content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, - spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, + content: %{"text/plain" => HTML.strip_tags(object_data["content"])}, + spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])}, expires_at: nil, direct_conversation_id: nil, thread_muted: false From 363e76d4dac290f5f5081e95ad40f496ee81c1e5 Mon Sep 17 00:00:00 2001 From: kPherox Date: Wed, 30 Oct 2019 15:40:25 +0900 Subject: [PATCH 21/39] Fix duplicate recipients --- lib/pleroma/user/query.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 2eda454bc..364bc1c89 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -175,6 +175,7 @@ defmodule Pleroma.User.Query do [u, following: f, relationships: r], u.ap_id in ^to or (f.follower_address in ^to and r.state == "accept") ) + |> distinct(true) end defp compose_query({:order_by, key}, query) do From 40d5fb6ef875262795d87b95bff1ef44ee75d659 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 30 Oct 2019 15:52:37 +0700 Subject: [PATCH 22/39] Add a migration to fix blocked follows --- .../20191029172832_fix_blocked_follows.exs | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 priv/repo/migrations/20191029172832_fix_blocked_follows.exs diff --git a/priv/repo/migrations/20191029172832_fix_blocked_follows.exs b/priv/repo/migrations/20191029172832_fix_blocked_follows.exs new file mode 100644 index 000000000..71f8f1330 --- /dev/null +++ b/priv/repo/migrations/20191029172832_fix_blocked_follows.exs @@ -0,0 +1,112 @@ +defmodule Pleroma.Repo.Migrations.FixBlockedFollows do + use Ecto.Migration + + import Ecto.Query + alias Pleroma.Config + alias Pleroma.Repo + + def up do + unfollow_blocked = Config.get([:activitypub, :unfollow_blocked]) + + if unfollow_blocked do + "activities" + |> where([activity], fragment("? ->> 'type' = 'Block'", activity.data)) + |> distinct([activity], [ + activity.actor, + fragment( + "coalesce((?)->'object'->>'id', (?)->>'object')", + activity.data, + activity.data + ) + ]) + |> order_by([activity], [fragment("? desc nulls last", activity.id)]) + |> select([activity], %{ + blocker: activity.actor, + blocked: + fragment("coalesce((?)->'object'->>'id', (?)->>'object')", activity.data, activity.data), + created_at: activity.id + }) + |> Repo.stream() + |> Enum.map(&unfollow_if_blocked/1) + |> Enum.uniq() + |> Enum.each(&update_follower_count/1) + end + end + + def down do + end + + def unfollow_if_blocked(%{blocker: blocker_id, blocked: blocked_id, created_at: blocked_at}) do + query = + from( + activity in "activities", + where: fragment("? ->> 'type' = 'Follow'", activity.data), + where: activity.actor == ^blocked_id, + # this is to use the index + where: + fragment( + "coalesce((?)->'object'->>'id', (?)->>'object') = ?", + activity.data, + activity.data, + ^blocker_id + ), + where: activity.id > ^blocked_at, + where: fragment("(?)->>'state' = 'accept'", activity.data), + order_by: [fragment("? desc nulls last", activity.id)] + ) + + unless Repo.exists?(query) do + blocker = "users" |> select([:id, :local]) |> Repo.get_by(ap_id: blocker_id) + blocked = "users" |> select([:id]) |> Repo.get_by(ap_id: blocked_id) + + if !is_nil(blocker) && !is_nil(blocked) do + unfollow(blocked, blocker) + end + end + end + + def unfollow(%{id: follower_id}, %{id: followed_id} = followed) do + following_relationship = + "following_relationships" + |> where(follower_id: ^follower_id, following_id: ^followed_id, state: "accept") + |> select([:id]) + |> Repo.one() + + case following_relationship do + nil -> + {:ok, nil} + + %{id: following_relationship_id} -> + "following_relationships" + |> where(id: ^following_relationship_id) + |> Repo.delete_all() + + followed + end + end + + def update_follower_count(%{id: user_id} = user) do + if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do + follower_count_query = + "users" + |> where([u], u.id != ^user_id) + |> where([u], u.deactivated != ^true) + |> join(:inner, [u], r in "following_relationships", + as: :relationships, + on: r.following_id == ^user_id and r.follower_id == u.id + ) + |> where([relationships: r], r.state == "accept") + |> select([u], %{count: count(u.id)}) + + "users" + |> where(id: ^user_id) + |> join(:inner, [u], s in subquery(follower_count_query)) + |> update([u, s], + set: [follower_count: s.count] + ) + |> Repo.update_all([]) + end + end + + def update_follower_count(_), do: :noop +end From c546da7cfe800ee0a0ac2ecf9981bb131cd63291 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 30 Oct 2019 12:59:14 +0300 Subject: [PATCH 23/39] Fix bookmark migration using a query with a schema This resulted in failures when updating from Pleroma <1.0 because of all the new fields that were added to the user schema. --- .../migrations/20190414125034_migrate_old_bookmarks.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs index f3928a149..99102117f 100644 --- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs +++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs @@ -8,10 +8,10 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do def up do query = - from(u in User, + from(u in "users", where: u.local == true, - where: fragment("array_length(bookmarks, 1)") > 0, - select: %{id: u.id, bookmarks: fragment("bookmarks")} + where: fragment("array_length(?, 1)", u.bookmarks) > 0, + select: %{id: u.id, bookmarks: u.bookmarks} ) Repo.stream(query) From d1ae51b02a8f367a5dec6d427391e274caeac468 Mon Sep 17 00:00:00 2001 From: kPherox Date: Wed, 30 Oct 2019 20:14:07 +0900 Subject: [PATCH 24/39] Update test for get_recipients_from_activity --- test/user_test.exs | 59 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/test/user_test.exs b/test/user_test.exs index 188295a86..6b1b24ce5 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -878,27 +878,50 @@ defmodule Pleroma.UserTest do end end - test "get recipients from activity" do - actor = insert(:user) - user = insert(:user, local: true) - user_two = insert(:user, local: false) - addressed = insert(:user, local: true) - addressed_remote = insert(:user, local: false) + describe "get_recipients_from_activity" do + test "get recipients" do + actor = insert(:user) + user = insert(:user, local: true) + user_two = insert(:user, local: false) + addressed = insert(:user, local: true) + addressed_remote = insert(:user, local: false) - {:ok, activity} = - CommonAPI.post(actor, %{ - "status" => "hey @#{addressed.nickname} @#{addressed_remote.nickname}" - }) + {:ok, activity} = + CommonAPI.post(actor, %{ + "status" => "hey @#{addressed.nickname} @#{addressed_remote.nickname}" + }) - assert Enum.map([actor, addressed], & &1.ap_id) -- - Enum.map(User.get_recipients_from_activity(activity), & &1.ap_id) == [] + assert Enum.map([actor, addressed], & &1.ap_id) -- + Enum.map(User.get_recipients_from_activity(activity), & &1.ap_id) == [] - {:ok, user} = User.follow(user, actor) - {:ok, _user_two} = User.follow(user_two, actor) - recipients = User.get_recipients_from_activity(activity) - assert length(recipients) == 3 - assert user in recipients - assert addressed in recipients + {:ok, user} = User.follow(user, actor) + {:ok, _user_two} = User.follow(user_two, actor) + recipients = User.get_recipients_from_activity(activity) + assert length(recipients) == 3 + assert user in recipients + assert addressed in recipients + end + + test "has following" do + actor = insert(:user) + user = insert(:user) + user_two = insert(:user) + addressed = insert(:user, local: true) + + {:ok, activity} = + CommonAPI.post(actor, %{ + "status" => "hey @#{addressed.nickname}" + }) + + assert Enum.map([actor, addressed], & &1.ap_id) -- + Enum.map(User.get_recipients_from_activity(activity), & &1.ap_id) == [] + + {:ok, _actor} = User.follow(actor, user) + {:ok, _actor} = User.follow(actor, user_two) + recipients = User.get_recipients_from_activity(activity) + assert length(recipients) == 2 + assert addressed in recipients + end end describe ".deactivate" do From 8bb6da7cd604359afb5eeabeeac0207edff07ce3 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 30 Oct 2019 18:34:14 +0300 Subject: [PATCH 25/39] Remove following column from the migrations --- priv/repo/migrations/20191025143434_add_defaults_to_tables.exs | 3 --- priv/repo/migrations/20191026191910_set_not_null_for_users.exs | 2 -- 2 files changed, 5 deletions(-) diff --git a/priv/repo/migrations/20191025143434_add_defaults_to_tables.exs b/priv/repo/migrations/20191025143434_add_defaults_to_tables.exs index d16ab19f9..a5bc82335 100644 --- a/priv/repo/migrations/20191025143434_add_defaults_to_tables.exs +++ b/priv/repo/migrations/20191025143434_add_defaults_to_tables.exs @@ -12,7 +12,6 @@ defmodule Pleroma.Repo.Migrations.AddDefaultsToTables do ALTER COLUMN data SET DEFAULT '{}'::jsonb") execute(~s(ALTER TABLE users - ALTER COLUMN following SET DEFAULT ARRAY[]::character varying[], ALTER COLUMN tags SET DEFAULT ARRAY[]::character varying[], ALTER COLUMN notification_settings SET DEFAULT '{"followers": true, "follows": true, "non_follows": true, "non_followers": true}'::jsonb)) @@ -46,7 +45,6 @@ defmodule Pleroma.Repo.Migrations.AddDefaultsToTables do "UPDATE users SET pleroma_settings_store = '{}'::jsonb where pleroma_settings_store IS NULL" ) - execute("UPDATE users SET following = ARRAY[]::character varying[] WHERE following IS NULL") execute("UPDATE users SET tags = ARRAY[]::character varying[] WHERE tags IS NULL") execute(~s(UPDATE users SET notification_settings = '{"followers": true, "follows": true, "non_follows": true, "non_followers": true}'::jsonb @@ -64,7 +62,6 @@ defmodule Pleroma.Repo.Migrations.AddDefaultsToTables do ALTER COLUMN data DROP DEFAULT") execute("ALTER TABLE users - ALTER COLUMN following DROP DEFAULT, ALTER COLUMN tags DROP DEFAULT, ALTER COLUMN notification_settings SET DEFAULT '{}'::jsonb") end diff --git a/priv/repo/migrations/20191026191910_set_not_null_for_users.exs b/priv/repo/migrations/20191026191910_set_not_null_for_users.exs index f145a89ab..9d8d0ccf8 100644 --- a/priv/repo/migrations/20191026191910_set_not_null_for_users.exs +++ b/priv/repo/migrations/20191026191910_set_not_null_for_users.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Repo.Migrations.SetNotNullForUsers do execute("UPDATE users SET follower_count = 0 WHERE follower_count IS NULL") execute("ALTER TABLE users - ALTER COLUMN following SET NOT NULL, ALTER COLUMN local SET NOT NULL, ALTER COLUMN source_data SET NOT NULL, ALTER COLUMN note_count SET NOT NULL, @@ -27,7 +26,6 @@ defmodule Pleroma.Repo.Migrations.SetNotNullForUsers do def down do execute("ALTER TABLE users - ALTER COLUMN following DROP NOT NULL, ALTER COLUMN local DROP NOT NULL, ALTER COLUMN source_data DROP NOT NULL, ALTER COLUMN note_count DROP NOT NULL, From bd0f7f2e09ec4d2883b87fd3c5d3784319de7cc7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 30 Oct 2019 17:57:08 -0500 Subject: [PATCH 26/39] Bump fast_sanitize dependency so this builds on more than just Linux --- mix.exs | 2 +- mix.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index e85c2aa50..66cea7b89 100644 --- a/mix.exs +++ b/mix.exs @@ -110,7 +110,7 @@ defmodule Pleroma.Mixfile do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", - ref: "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c"}, + ref: "d5c47d580a018c4e7f6fadfca63834b9ae28c18c"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, diff --git a/mix.lock b/mix.lock index 6cbe54a76..802cb2959 100644 --- a/mix.lock +++ b/mix.lock @@ -36,7 +36,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c", [ref: "37f55e2bbe6eb44e8a9c4b3831a5ce73a2f6955c"]}, + "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "d5c47d580a018c4e7f6fadfca63834b9ae28c18c", [ref: "d5c47d580a018c4e7f6fadfca63834b9ae28c18c"]}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, @@ -64,9 +64,9 @@ "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, - "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "eafbba443260c0055ef2da3918b351b112ff356f", [ref: "eafbba443260c0055ef2da3918b351b112ff356f", submodules: true]}, + "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "99140014f5da4081d2db437e6d769b81c70bf510", [ref: "99140014f5da4081d2db437e6d769b81c70bf510", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, - "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "2927091d96900fb76f6bc897e46a6abb9070ebbd", [ref: "2927091d96900fb76f6bc897e46a6abb9070ebbd"]}, + "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, From 59a149c69a9a6726c7687ba233564936e47fc199 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 31 Oct 2019 02:25:15 +0300 Subject: [PATCH 27/39] Fix "the call ... will never return" warning --- lib/pleroma/web/admin_api/search.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/search.ex b/lib/pleroma/web/admin_api/search.ex index ed919833e..778cf4c36 100644 --- a/lib/pleroma/web/admin_api/search.ex +++ b/lib/pleroma/web/admin_api/search.ex @@ -18,7 +18,11 @@ defmodule Pleroma.Web.AdminAPI.Search do @spec user(map()) :: {:ok, [User.t()], pos_integer()} def user(params \\ %{}) do - query = User.Query.build(params) |> order_by([u], u.nickname) + query = + params + |> Map.drop([:page, :page_size]) + |> User.Query.build() + |> order_by([u], u.nickname) paginated_query = User.Query.paginate(query, params[:page] || 1, params[:page_size] || @page_size) From 6f9d3d30faece1432068a421fd74d68d93e1d313 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 31 Oct 2019 02:26:02 +0300 Subject: [PATCH 28/39] AdminAPI: Omit relay user from users list --- lib/pleroma/web/activity_pub/relay.ex | 6 ++++- .../web/admin_api/admin_api_controller.ex | 9 ++++++- .../admin_api/admin_api_controller_test.exs | 27 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index f90d75a8a..fc2619680 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -11,13 +11,17 @@ defmodule Pleroma.Web.ActivityPub.Relay do def get_actor do actor = - "#{Pleroma.Web.Endpoint.url()}/relay" + relay_ap_id() |> User.get_or_create_service_actor_by_ap_id() {:ok, actor} = User.set_invisible(actor, true) actor end + def relay_ap_id do + "#{Pleroma.Web.Endpoint.url()}/relay" + end + @spec follow(String.t()) :: {:ok, Activity.t()} | {:error, any()} def follow(target_instance) do with %User{} = local_user <- get_actor(), diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 7ffbb23e7..4533d0114 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -333,7 +333,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do email: params["email"] } - with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)), + with {:ok, users, _count} <- Search.user(Map.merge(search_params, filters)), + {:ok, users, count} <- filter_relay_user(users), do: conn |> json( @@ -345,6 +346,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do ) end + defp filter_relay_user(users) do + filtered_users = Enum.reject(users, &(&1.ap_id == Relay.relay_ap_id())) + + {:ok, filtered_users, length(filtered_users)} + end + @filters ~w(local external active deactivated is_admin is_moderator) @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{} diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 22c989892..045c87e95 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.UserInviteToken + alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.CommonAPI alias Pleroma.Web.MediaProxy import Pleroma.Factory @@ -1044,6 +1045,32 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do ] } end + + test "it omits relay user", %{admin: admin} do + assert %User{} = Relay.get_actor() + + conn = + build_conn() + |> assign(:user, admin) + |> get("/api/pleroma/admin/users") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [ + %{ + "deactivated" => admin.deactivated, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => true, + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname) + } + ] + } + end end test "PATCH /api/pleroma/admin/users/activate" do From ced9f923270e6b30c4b19a83a8f37516c0e49cf6 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 31 Oct 2019 15:34:49 +0300 Subject: [PATCH 29/39] Fix count --- lib/pleroma/web/admin_api/admin_api_controller.ex | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 4533d0114..b47618bde 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -333,8 +333,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do email: params["email"] } - with {:ok, users, _count} <- Search.user(Map.merge(search_params, filters)), - {:ok, users, count} <- filter_relay_user(users), + with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)), + {:ok, users, count} <- filter_relay_user(users, count), do: conn |> json( @@ -346,10 +346,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do ) end - defp filter_relay_user(users) do - filtered_users = Enum.reject(users, &(&1.ap_id == Relay.relay_ap_id())) + defp filter_relay_user(users, count) do + filtered_users = Enum.reject(users, &relay_user?/1) + count = if Enum.any?(users, &relay_user?/1), do: length(filtered_users), else: count - {:ok, filtered_users, length(filtered_users)} + {:ok, filtered_users, count} + end + + defp relay_user?(user) do + user.ap_id == Relay.relay_ap_id() end @filters ~w(local external active deactivated is_admin is_moderator) From d75934b0d024296654a7eec74abcd65832b6b96b Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 1 Nov 2019 15:14:43 +0300 Subject: [PATCH 30/39] Undo dialyzer fix --- lib/pleroma/web/admin_api/search.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/pleroma/web/admin_api/search.ex b/lib/pleroma/web/admin_api/search.ex index 778cf4c36..ed919833e 100644 --- a/lib/pleroma/web/admin_api/search.ex +++ b/lib/pleroma/web/admin_api/search.ex @@ -18,11 +18,7 @@ defmodule Pleroma.Web.AdminAPI.Search do @spec user(map()) :: {:ok, [User.t()], pos_integer()} def user(params \\ %{}) do - query = - params - |> Map.drop([:page, :page_size]) - |> User.Query.build() - |> order_by([u], u.nickname) + query = User.Query.build(params) |> order_by([u], u.nickname) paginated_query = User.Query.paginate(query, params[:page] || 1, params[:page_size] || @page_size) From 1b83a0694a19e279d155dde2c915df3583f12170 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 1 Nov 2019 19:13:29 +0300 Subject: [PATCH 31/39] Fix moderation log crash --- lib/pleroma/moderation_log.ex | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index e8884e6e8..9dc4a94c9 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -369,6 +369,24 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}" end + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "activate", + "subject" => user + } + }) + when is_map(user) do + get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "activate", + "subject" => [user] + } + }) + end + @spec get_log_entry_message(ModerationLog) :: String.t() def get_log_entry_message(%ModerationLog{ data: %{ @@ -380,6 +398,24 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}" end + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "deactivate", + "subject" => user + } + }) + when is_map(user) do + get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "deactivate", + "subject" => [user] + } + }) + end + @spec get_log_entry_message(ModerationLog) :: String.t() def get_log_entry_message(%ModerationLog{ data: %{ @@ -419,6 +455,26 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}" end + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "grant", + "subject" => user, + "permission" => permission + } + }) + when is_map(user) do + get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "grant", + "subject" => [user], + "permission" => permission + } + }) + end + @spec get_log_entry_message(ModerationLog) :: String.t() def get_log_entry_message(%ModerationLog{ data: %{ @@ -431,6 +487,26 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}" end + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "revoke", + "subject" => user, + "permission" => permission + } + }) + when is_map(user) do + get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "revoke", + "subject" => [user], + "permission" => permission + } + }) + end + @spec get_log_entry_message(ModerationLog) :: String.t() def get_log_entry_message(%ModerationLog{ data: %{ From 55ef88ef95ecda94ea416bd3a8492f992488ea83 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 1 Nov 2019 20:58:24 +0300 Subject: [PATCH 32/39] Bump fast_sanitize version --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index 66cea7b89..f3ef8d955 100644 --- a/mix.exs +++ b/mix.exs @@ -110,7 +110,7 @@ defmodule Pleroma.Mixfile do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", - ref: "d5c47d580a018c4e7f6fadfca63834b9ae28c18c"}, + ref: "cb92cc6f63b183863f5cecb0c740c34642da5216"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, diff --git a/mix.lock b/mix.lock index 802cb2959..c867a9605 100644 --- a/mix.lock +++ b/mix.lock @@ -36,7 +36,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "d5c47d580a018c4e7f6fadfca63834b9ae28c18c", [ref: "d5c47d580a018c4e7f6fadfca63834b9ae28c18c"]}, + "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "cb92cc6f63b183863f5cecb0c740c34642da5216", [ref: "cb92cc6f63b183863f5cecb0c740c34642da5216"]}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, @@ -64,7 +64,7 @@ "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, - "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "99140014f5da4081d2db437e6d769b81c70bf510", [ref: "99140014f5da4081d2db437e6d769b81c70bf510", submodules: true]}, + "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "4288bda52c53a2d416afa09e1007a881eb2dafab", [ref: "4288bda52c53a2d416afa09e1007a881eb2dafab", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, From 4bf942583fdae27813f4af1f901c78eaff391b76 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 3 Nov 2019 09:05:12 -0600 Subject: [PATCH 33/39] streamer: use direct object for filter checks when there is no valid child object in an activity We call Object.normalize/1 to get the child object for situations like Announce. However, the check is flawed and immediately fails if Object.normalize/1 fails. Instead, we should use the activity itself in those cases to allow activities which never have a child object to pass through the filter. Closes #1291 --- lib/pleroma/web/streamer/worker.ex | 2 +- test/web/streamer/streamer_test.exs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex index c2ee9e1f5..33b24840d 100644 --- a/lib/pleroma/web/streamer/worker.ex +++ b/lib/pleroma/web/streamer/worker.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Web.Streamer.Worker do recipients = MapSet.new(item.recipients) domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.domain_blocks) - with parent when not is_nil(parent) <- Object.normalize(item), + with parent <- Object.normalize(item) || item, true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), true <- MapSet.disjoint?(recipients, recipient_blocks), diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index cb1015171..d6968f4d0 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -110,6 +110,24 @@ defmodule Pleroma.Web.StreamerTest do Streamer.stream("user:notification", notif) Task.await(task) end + + test "it sends follow activities to the 'user:notification' stream", %{ + user: user + } do + user2 = insert(:user) + task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) + + Streamer.add_socket( + "user:notification", + %{transport_pid: task.pid, assigns: %{user: user}} + ) + + {:ok, _follower, _followed, _activity} = CommonAPI.follow(user2, user) + + # We don't directly pipe the notification to the streamer as it's already + # generated as a side effect of CommonAPI.follow(). + Task.await(task) + end end test "it sends to public" do From f42f30087049bb657ebbb8cd8511b1c6888e1ccd Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 3 Nov 2019 15:41:30 -0600 Subject: [PATCH 34/39] update fast sanitize dependency --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index f3ef8d955..f3db83556 100644 --- a/mix.exs +++ b/mix.exs @@ -110,7 +110,7 @@ defmodule Pleroma.Mixfile do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", - ref: "cb92cc6f63b183863f5cecb0c740c34642da5216"}, + ref: "757e816bbd2cdd3f8357d02a3ed11230e9ff80f2"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, From 0c3125861619f164015ee0cf0bdf293d49804926 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 4 Nov 2019 14:36:54 +0100 Subject: [PATCH 35/39] User: Don't pull remote users follower count immediately after deactivating. The other instance doesn't necessarily know that anything changed yet, and it will be fixed up at the next user pull anyway. Closes #1369 --- lib/pleroma/user.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 40171620e..f8c2db1e1 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1095,7 +1095,12 @@ defmodule Pleroma.User do def deactivate(%User{} = user, status) do with {:ok, user} <- set_activation_status(user, status) do Enum.each(get_followers(user), &invalidate_cache/1) - Enum.each(get_friends(user), &update_follower_count/1) + + # Only update local user counts, remote will be update during the next pull. + user + |> get_friends() + |> Enum.filter(& &1.local) + |> Enum.each(&update_follower_count/1) {:ok, user} end From 203d9812f0d14a0edd80871a1046a7eeaea6f606 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Mon, 4 Nov 2019 08:38:14 -0600 Subject: [PATCH 36/39] mix: update fast_sanitize dependency to 1af67547 --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index f3db83556..1bc4cec2f 100644 --- a/mix.exs +++ b/mix.exs @@ -110,7 +110,7 @@ defmodule Pleroma.Mixfile do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, git: "https://git.pleroma.social/pleroma/fast_sanitize.git", - ref: "757e816bbd2cdd3f8357d02a3ed11230e9ff80f2"}, + ref: "1af67547a02a104e26c99d03012383e8643bc4c2"}, {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, diff --git a/mix.lock b/mix.lock index c867a9605..cfc3b84a8 100644 --- a/mix.lock +++ b/mix.lock @@ -36,7 +36,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "cb92cc6f63b183863f5cecb0c740c34642da5216", [ref: "cb92cc6f63b183863f5cecb0c740c34642da5216"]}, + "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "1af67547a02a104e26c99d03012383e8643bc4c2", [ref: "1af67547a02a104e26c99d03012383e8643bc4c2"]}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, @@ -64,7 +64,7 @@ "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, - "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "4288bda52c53a2d416afa09e1007a881eb2dafab", [ref: "4288bda52c53a2d416afa09e1007a881eb2dafab", submodules: true]}, + "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, From 5271bbcf11d7182c25c8ca06460823e00920e80d Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Mon, 4 Nov 2019 15:18:32 +0000 Subject: [PATCH 37/39] add missing tesla mocks --- lib/pleroma/web/rel_me.ex | 14 +++++++------- test/support/http_request_mock.ex | 28 ++++++++++++++++++++++++++++ test/web/rel_me_test.exs | 4 +++- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index d376e2069..16b1a53d2 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -25,13 +25,13 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) - - data = - Floki.attribute(html, "link[rel~=me]", "href") ++ - Floki.attribute(html, "a[rel~=me]", "href") - - {:ok, data} + with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- + Pleroma.HTTP.get(url, [], adapter: @hackney_options), + data <- + Floki.attribute(html, "link[rel~=me]", "href") ++ + Floki.attribute(html, "a[rel~=me]", "href") do + {:ok, data} + end rescue e -> {:error, "Parsing error: #{inspect(e)}"} end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index eba22c40b..965335e96 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1183,6 +1183,30 @@ defmodule HttpRequestMock do }} end + def get("https://10.111.10.1/notice/9kCP7V", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: ""}} + end + + def get("https://172.16.32.40/notice/9kCP7V", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: ""}} + end + + def get("https://192.168.10.40/notice/9kCP7V", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: ""}} + end + + def get("https://www.patreon.com/posts/mastodon-2-9-and-28121681", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: ""}} + end + + def get("http://mastodon.example.org/@admin/99541947525187367", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/mastodon-post-activity.json")}} + end + + def get("https://info.pleroma.site/activity4.json", _, _, _) do + {:ok, %Tesla.Env{status: 500, body: "Error occurred"}} + end + def get("http://example.com/rel_me/anchor", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rel_me_anchor.html")}} end @@ -1215,6 +1239,10 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/rin.json")}} end + def get("http://example.com/rel_me/error", _, _, _) do + {:ok, %Tesla.Env{status: 404, body: ""}} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{ diff --git a/test/web/rel_me_test.exs b/test/web/rel_me_test.exs index 2251fed16..77b5d5dc6 100644 --- a/test/web/rel_me_test.exs +++ b/test/web/rel_me_test.exs @@ -14,7 +14,9 @@ defmodule Pleroma.Web.RelMeTest do hrefs = ["https://social.example.org/users/lain"] assert Pleroma.Web.RelMe.parse("http://example.com/rel_me/null") == {:ok, []} - assert {:error, _} = Pleroma.Web.RelMe.parse("http://example.com/rel_me/error") + + assert {:ok, %Tesla.Env{status: 404}} = + Pleroma.Web.RelMe.parse("http://example.com/rel_me/error") assert Pleroma.Web.RelMe.parse("http://example.com/rel_me/link") == {:ok, hrefs} assert Pleroma.Web.RelMe.parse("http://example.com/rel_me/anchor") == {:ok, hrefs} From ed29be24cbdc029614557a5289a9b8c8facddf8e Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 31 Oct 2019 03:44:27 +0300 Subject: [PATCH 38/39] Mastodon API, streaming: Add `pleroma.direct_conversation_id` to the `conversation` stream event payload. --- CHANGELOG.md | 1 + .../mastodon_api/views/conversation_view.ex | 6 ++++- .../web/mastodon_api/views/status_view.ex | 6 ++++- .../views/conversation_view_test.exs | 1 + .../mastodon_api/views/status_view_test.exs | 25 +++++++++++++++++-- test/web/streamer/streamer_test.exs | 12 +++++++-- 6 files changed, 45 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00dd62e24..77edf7bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) - Mastodon API: `pleroma.thread_muted` to the Status entity - Mastodon API: Mark the direct conversation as read for the author when they send a new direct message +- Mastodon API, streaming: Add `pleroma.direct_conversation_id` to the `conversation` stream event payload. ### Added diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index e9d2735b3..c5998e661 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -34,7 +34,11 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do id: participation.id |> to_string(), accounts: render(AccountView, "index.json", users: users, as: :user), unread: !participation.read, - last_status: render(StatusView, "show.json", activity: activity, for: user) + last_status: + render(StatusView, "show.json", + activity: activity, + direct_conversation_id: participation.id + ) } end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b785ca9d4..baff54151 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -243,7 +243,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end direct_conversation_id = - with {_, true} <- {:include_id, opts[:with_direct_conversation_id]}, + with {_, nil} <- {:direct_conversation_id, opts[:direct_conversation_id]}, + {_, true} <- {:include_id, opts[:with_direct_conversation_id]}, {_, %User{} = for_user} <- {:for_user, opts[:for]}, %{data: %{"context" => context}} when is_binary(context) <- activity, %Conversation{} = conversation <- Conversation.get_for_ap_id(context), @@ -251,6 +252,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do Participation.for_user_and_conversation(for_user, conversation) do participation_id else + {:direct_conversation_id, participation_id} when is_integer(participation_id) -> + participation_id + _e -> nil end diff --git a/test/web/mastodon_api/views/conversation_view_test.exs b/test/web/mastodon_api/views/conversation_view_test.exs index a2a880705..6ed22597d 100644 --- a/test/web/mastodon_api/views/conversation_view_test.exs +++ b/test/web/mastodon_api/views/conversation_view_test.exs @@ -30,5 +30,6 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do assert [account] = conversation.accounts assert account.id == other_user.id + assert conversation.last_status.pleroma.direct_conversation_id == participation.id end end diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index 7526f2f27..d46ecc646 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Conversation.Participation alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -23,10 +24,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do :ok end - test "returns the direct conversation id when given the `with_conversation_id` option" do + test "loads and returns the direct conversation id when given the `with_direct_conversation_id` option" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + [participation] = Participation.for_user(user) status = StatusView.render("show.json", @@ -35,7 +37,26 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do for: user ) - assert status[:pleroma][:direct_conversation_id] + assert status[:pleroma][:direct_conversation_id] == participation.id + + status = StatusView.render("show.json", activity: activity, for: user) + assert status[:pleroma][:direct_conversation_id] == nil + end + + test "returns the direct conversation id when given the `direct_conversation_id` option" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + [participation] = Participation.for_user(user) + + status = + StatusView.render("show.json", + activity: activity, + direct_conversation_id: participation.id, + for: user + ) + + assert status[:pleroma][:direct_conversation_id] == participation.id end test "returns a temporary ap_id based user for activities missing db users" do diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index d6968f4d0..80a7541b2 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.StreamerTest do import Pleroma.Factory + alias Pleroma.Conversation.Participation alias Pleroma.List alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -481,7 +482,14 @@ defmodule Pleroma.Web.StreamerTest do task = Task.async(fn -> - assert_receive {:text, _received_event}, 4_000 + assert_receive {:text, received_event}, 4_000 + + assert %{"event" => "conversation", "payload" => received_payload} = + Jason.decode!(received_event) + + assert %{"last_status" => last_status} = Jason.decode!(received_payload) + [participation] = Participation.for_user(user) + assert last_status["pleroma"]["direct_conversation_id"] == participation.id end) Streamer.add_socket( @@ -498,7 +506,7 @@ defmodule Pleroma.Web.StreamerTest do Task.await(task) end - test "it doesn't send conversation update to the 'direct' streamj when the last message in the conversation is deleted" do + test "it doesn't send conversation update to the 'direct' stream when the last message in the conversation is deleted" do user = insert(:user) another_user = insert(:user) From 4e535209172bb5460353fe011c06d127cfaa5847 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 4 Nov 2019 16:57:41 +0100 Subject: [PATCH 39/39] User Search: Remove superfluous setweight and random test. The test tested for a behavior that isn't actually enforced anymore. --- lib/pleroma/user/search.ex | 12 +----------- test/user_search_test.exs | 7 ------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index bab8d92e2..09664db76 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -54,15 +54,7 @@ defmodule Pleroma.User.Search do |> maybe_restrict_local(for_user) end - @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/ defp fts_search(query, query_string) do - {nickname_weight, name_weight} = - if String.match?(query_string, @nickname_regex) do - {"A", "B"} - else - {"B", "A"} - end - query_string = to_tsquery(query_string) from( @@ -70,12 +62,10 @@ defmodule Pleroma.User.Search do where: fragment( """ - (setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)) @@ to_tsquery('simple', ?) + (to_tsvector('simple', ?) || to_tsvector('simple', ?)) @@ to_tsquery('simple', ?) """, u.name, - ^name_weight, u.nickname, - ^nickname_weight, ^query_string ) ) diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 78a02d536..721af1e5b 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -51,13 +51,6 @@ defmodule Pleroma.UserSearchTest do end) end - test "finds users, preferring nickname matches over name matches" do - u1 = insert(:user, %{name: "lain", nickname: "nick1"}) - u2 = insert(:user, %{nickname: "lain", name: "nick1"}) - - assert [u2.id, u1.id] == Enum.map(User.search("lain"), & &1.id) - end - test "finds users, considering density of matched tokens" do u1 = insert(:user, %{name: "Bar Bar plus Word Word"}) u2 = insert(:user, %{name: "Word Word Bar Bar Bar"})