Fix fetching of community posts (fixes #4283) (#4293)

* Fix fetching of community posts (fixes #4283)

Also use spawn_try_task to fetch community outbox, mods etc to avoid
delay/timeout when fetching new community.

* prettier

* fix test

* fix api test

* prettier

* add delay

* Update run-federation-test.sh

* fix test

api_tests/src/community.spec.ts

@@ -31,6 +31,7 @@ import {
   searchPostLocal,
   resolveBetaCommunity,
   longDelay,
+  delay,
 } from "./shared";
 import { EditSite } from "lemmy-js-client";
 
@@ -377,7 +378,9 @@ test("User blocks instance, communities are hidden", async () => {
 
 test("Community follower count is federated", async () => {
   // Follow the beta community from alpha
-  let resolved = await resolveBetaCommunity(alpha);
+  let community = await createCommunity(beta);
+  let community_id = community.community_view.community.actor_id;
+  let resolved = await resolveCommunity(alpha, community_id);
   if (!resolved.community) {
     throw "Missing beta community";
   }
@@ -385,7 +388,7 @@ test("Community follower count is federated", async () => {
   await followCommunity(alpha, true, resolved.community.community.id);
   let followed = (
     await waitUntil(
-      () => resolveBetaCommunity(alpha),
+      () => resolveCommunity(alpha, community_id),
       c => c.community?.subscribed === "Subscribed",
     )
   ).community;
@@ -394,7 +397,7 @@ test("Community follower count is federated", async () => {
   expect(followed?.counts.subscribers).toBe(1);
 
   // Follow the community from gamma
-  resolved = await resolveBetaCommunity(gamma);
+  resolved = await resolveCommunity(gamma, community_id);
   if (!resolved.community) {
     throw "Missing beta community";
   }
@@ -402,7 +405,7 @@ test("Community follower count is federated", async () => {
   await followCommunity(gamma, true, resolved.community.community.id);
   followed = (
     await waitUntil(
-      () => resolveBetaCommunity(gamma),
+      () => resolveCommunity(gamma, community_id),
       c => c.community?.subscribed === "Subscribed",
     )
   ).community;
@@ -411,7 +414,7 @@ test("Community follower count is federated", async () => {
   expect(followed?.counts?.subscribers).toBe(2);
 
   // Follow the community from delta
-  resolved = await resolveBetaCommunity(delta);
+  resolved = await resolveCommunity(delta, community_id);
   if (!resolved.community) {
     throw "Missing beta community";
   }
@@ -419,7 +422,7 @@ test("Community follower count is federated", async () => {
   await followCommunity(delta, true, resolved.community.community.id);
   followed = (
     await waitUntil(
-      () => resolveBetaCommunity(delta),
+      () => resolveCommunity(delta, community_id),
       c => c.community?.subscribed === "Subscribed",
     )
   ).community;
@@ -480,3 +483,31 @@ test("Dont receive community activities after unsubscribe", async () => {
   let postResBeta = searchPostLocal(beta, postRes.post_view.post);
   expect((await postResBeta).posts.length).toBe(0);
 });
+
+test("Fetch community, includes posts", async () => {
+  let communityRes = await createCommunity(alpha);
+  expect(communityRes.community_view.community.name).toBeDefined();
+  expect(communityRes.community_view.counts.subscribers).toBe(1);
+
+  let postRes = await createPost(
+    alpha,
+    communityRes.community_view.community.id,
+  );
+  expect(postRes.post_view.post).toBeDefined();
+
+  let resolvedCommunity = await waitUntil(
+    () =>
+      resolveCommunity(beta, communityRes.community_view.community.actor_id),
+    c => c.community?.community.id != undefined,
+  );
+  let betaCommunity = resolvedCommunity.community;
+  expect(betaCommunity?.community.actor_id).toBe(
+    communityRes.community_view.community.actor_id,
+  );
+
+  await longDelay();
+
+  let post_listing = await getPosts(beta, "All", betaCommunity?.community.id);
+  expect(post_listing.posts.length).toBe(1);
+  expect(post_listing.posts[0].post.ap_id).toBe(postRes.post_view.post.ap_id);
+});

api_tests/src/shared.ts

@@ -805,10 +805,12 @@ export async function listCommentReports(
 export function getPosts(
   api: LemmyHttp,
   listingType?: ListingType,
+  community_id?: number,
 ): Promise<GetPostsResponse> {
   let form: GetPosts = {
     type_: listingType,
     limit: 50,
+    community_id,
   };
   return api.getPosts(form);
 }

crates/apub/src/collections/community_outbox.rs

@@ -96,11 +96,15 @@ impl Collection for ApubCommunityOutbox {
     // process items in parallel, to avoid long delay from fetch_site_metadata() and other processing
     join_all(outbox_activities.into_iter().map(|activity| {
       async {
-        // use separate request counter for each item, otherwise there will be problems with
-        // parallel processing
-        let verify = activity.verify(data).await;
+        // Receiving announce requires at least one local community follower for anti spam purposes.
+        // This won't be the case for newly fetched communities, so we extract the inner activity
+        // and handle it directly to bypass this check.
+        let inner = activity.object.object(data).await.map(TryInto::try_into);
+        if let Ok(Ok(AnnouncableActivities::CreateOrUpdatePost(inner))) = inner {
+          let verify = inner.verify(data).await;
           if verify.is_ok() {
-          activity.receive(data).await.ok();
+            inner.receive(data).await.ok();
+          }
         }
       }
     }))

crates/apub/src/objects/community.rs

@@ -28,9 +28,8 @@ use lemmy_db_schema::{
   traits::{ApubActor, Crud},
 };
 use lemmy_db_views_actor::structs::CommunityFollowerView;
-use lemmy_utils::{error::LemmyError, utils::markdown::markdown_to_html};
+use lemmy_utils::{error::LemmyError, spawn_try_task, utils::markdown::markdown_to_html};
 use std::ops::Deref;
-use tracing::debug;
 use url::Url;
 
 #[derive(Clone, Debug)]
@@ -142,21 +141,16 @@ impl Object for ApubCommunity {
 
     // Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides,
     // we need to ignore these errors so that tests can work entirely offline.
-    let fetch_outbox = group.outbox.dereference(&community, context);
-    let fetch_followers = group.followers.dereference(&community, context);
-
+    let community_ = community.clone();
+    let context_ = context.reset_request_count();
+    spawn_try_task(async move {
+      group.outbox.dereference(&community_, &context_).await?;
+      group.followers.dereference(&community_, &context_).await?;
       if let Some(moderators) = group.attributed_to {
-      let fetch_moderators = moderators.dereference(&community, context);
-      // Fetch mods, outbox and followers in parallel
-      let res = tokio::join!(fetch_outbox, fetch_moderators, fetch_followers);
-      res.0.map_err(|e| debug!("{}", e)).ok();
-      res.1.map_err(|e| debug!("{}", e)).ok();
-      res.2.map_err(|e| debug!("{}", e)).ok();
-    } else {
-      let res = tokio::join!(fetch_outbox, fetch_followers);
-      res.0.map_err(|e| debug!("{}", e)).ok();
-      res.1.map_err(|e| debug!("{}", e)).ok();
+        moderators.dereference(&community_, &context_).await?;
       }
+      Ok(())
+    });
 
     Ok(community)
   }
@@ -241,8 +235,6 @@ pub(crate) mod tests {
     let url = Url::parse("https://enterprise.lemmy.ml/c/tenforward")?;
     ApubCommunity::verify(&json, &url, &context2).await?;
     let community = ApubCommunity::from_json(json, &context2).await?;
-    // this makes requests to the (intentionally broken) outbox and followers collections
-    assert_eq!(context2.request_count(), 2);
     Ok(community)
   }