Post creation from Mastodon (fixes #2590) (#2651)

* Post creation from Mastodon (fixes #2590)

* better logic for page title

* add deserialize helper

Co-authored-by: Dessalines <dessalines@users.noreply.github.com>

crates/apub/assets/mastodon/objects/page.json

@@ -0,0 +1,53 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "https://mastodon.madrid/users/felix/statuses/107224289116410645",
+  "type": "Note",
+  "summary": null,
+  "published": "2021-11-05T11:46:50Z",
+  "url": "https://mastodon.madrid/@felix/107224289116410645",
+  "attributedTo": "https://mastodon.madrid/users/felix",
+  "to": [
+    "https://mastodon.madrid/users/felix/followers"
+  ],
+  "cc": [
+    "https://www.w3.org/ns/activitystreams#Public",
+    "https://mamot.fr/users/retiolus"
+  ],
+  "sensitive": false,
+  "atomUri": "https://mastodon.madrid/users/felix/statuses/107224289116410645",
+  "inReplyToAtomUri": "https://mamot.fr/users/retiolus/statuses/107224244380204526",
+  "conversation": "tag:mamot.fr,2021-11-05:objectId=64635960:objectType=Conversation",
+  "content": "<p><span class=\"h-card\"><a href=\"https://mamot.fr/@retiolus\" class=\"u-url mention\">@<span>retiolus</span></a></span> i have never been disappointed by a thinkpad. if you want to save money, get a model from a few years ago, there isnt a huge difference anyway.</p>",
+  "contentMap": {
+    "en": "<p><span class=\"h-card\"><a href=\"https://mamot.fr/@retiolus\" class=\"u-url mention\">@<span>retiolus</span></a></span> i have never been disappointed by a thinkpad. if you want to save money, get a model from a few years ago, there isnt a huge difference anyway.</p>"
+  },
+  "attachment": [],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://mamot.fr/users/retiolus",
+      "name": "@retiolus@mamot.fr"
+    }
+  ],
+  "replies": {
+    "id": "https://mastodon.madrid/users/felix/statuses/107224289116410645/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "https://mastodon.madrid/users/felix/statuses/107224289116410645/replies?only_other_accounts=true&page=true",
+      "partOf": "https://mastodon.madrid/users/felix/statuses/107224289116410645/replies",
+      "items": []
+    }
+  }
+}

crates/apub/src/api/resolve_object.rs

@@ -46,12 +46,7 @@ async fn convert_response(
 ) -> Result<ResolveObjectResponse, LemmyError> {
   use SearchableObjects::*;
   let removed_or_deleted;
-  let mut res = ResolveObjectResponse {
-    comment: None,
-    post: None,
-    community: None,
-    person: None,
-  };
+  let mut res = ResolveObjectResponse::default();
   match object {
     Person(p) => {
       removed_or_deleted = p.deleted;

crates/apub/src/objects/post.rs

@@ -21,6 +21,7 @@ use activitypub_federation::{
   utils::verify_domains_match,
 };
 use activitystreams_kinds::public;
+use anyhow::anyhow;
 use chrono::NaiveDateTime;
 use lemmy_api_common::{
   context::LemmyContext,
@@ -40,11 +41,13 @@ use lemmy_db_schema::{
 };
 use lemmy_utils::{
   error::LemmyError,
-  utils::{check_slurs, convert_datetime, markdown_to_html, remove_slurs},
+  utils::{check_slurs_opt, convert_datetime, markdown_to_html, remove_slurs},
 };
 use std::ops::Deref;
 use url::Url;
 
+const MAX_TITLE_LENGTH: usize = 100;
+
 #[derive(Clone, Debug)]
 pub struct ApubPost(pub(crate) Post);
 
@@ -108,7 +111,7 @@ impl ApubObject for ApubPost {
       attributed_to: AttributedTo::Lemmy(ObjectId::new(creator.actor_id)),
       to: vec![community.actor_id.clone().into(), public()],
       cc: vec![],
-      name: self.name.clone(),
+      name: Some(self.name.clone()),
       content: self.body.as_ref().map(|b| markdown_to_html(b)),
       media_type: Some(MediaTypeMarkdownOrHtml::Html),
       source: self.body.clone().map(Source::new),
@@ -121,6 +124,7 @@ impl ApubObject for ApubPost {
       published: Some(convert_datetime(self.published)),
       updated: self.updated.map(convert_datetime),
       audience: Some(ObjectId::new(community.actor_id)),
+      in_reply_to: None,
     };
     Ok(page)
   }
@@ -151,7 +155,7 @@ impl ApubObject for ApubPost {
     verify_person_in_community(&page.creator()?, &community, context, request_counter).await?;
 
     let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site);
-    check_slurs(&page.name, slur_regex)?;
+    check_slurs_opt(&page.name, slur_regex)?;
 
     verify_domains_match(page.creator()?.inner(), page.id.inner())?;
     verify_is_public(&page.to, &page.cc)?;
@@ -169,6 +173,19 @@ impl ApubObject for ApubPost {
       .dereference(context, local_instance(context).await, request_counter)
       .await?;
     let community = page.community(context, request_counter).await?;
+    let mut name = page
+      .name
+      .clone()
+      .or_else(|| {
+        page
+          .content
+          .clone()
+          .and_then(|c| c.lines().next().map(ToString::to_string))
+      })
+      .ok_or_else(|| anyhow!("Object must have name or content"))?;
+    if name.chars().count() > MAX_TITLE_LENGTH {
+      name = name.chars().take(MAX_TITLE_LENGTH).collect();
+    }
 
     let form = if !page.is_mod_action(context).await? {
       let first_attachment = page.attachment.into_iter().map(Attachment::url).next();
@@ -197,7 +214,7 @@ impl ApubObject for ApubPost {
       let language_id = LanguageTag::to_language_id_single(page.language, context.pool()).await?;
 
       PostInsertForm {
-        name: page.name.clone(),
+        name,
         url: url.map(Into::into),
         body: body_slurs_removed,
         creator_id: creator.id,
@@ -221,7 +238,7 @@ impl ApubObject for ApubPost {
     } else {
       // if is mod action, only update locked/stickied fields, nothing else
       PostInsertForm::builder()
-        .name(page.name.clone())
+        .name(name)
         .creator_id(creator.id)
         .community_id(community.id)
         .ap_id(Some(page.id.clone().into()))

crates/apub/src/protocol/objects/mod.rs

@@ -131,6 +131,7 @@ mod tests {
   fn test_parse_objects_mastodon() {
     test_json::<Person>("assets/mastodon/objects/person.json").unwrap();
     test_json::<Note>("assets/mastodon/objects/note.json").unwrap();
+    test_json::<Page>("assets/mastodon/objects/page.json").unwrap();
   }
 
   #[test]

crates/apub/src/protocol/objects/page.rs

@@ -23,7 +23,7 @@ use itertools::Itertools;
 use lemmy_api_common::context::LemmyContext;
 use lemmy_db_schema::newtypes::DbUrl;
 use lemmy_utils::error::LemmyError;
-use serde::{Deserialize, Serialize};
+use serde::{de::Error, Deserialize, Deserializer, Serialize};
 use serde_with::skip_serializing_none;
 use url::Url;
 
@@ -46,8 +46,11 @@ pub struct Page {
   pub(crate) attributed_to: AttributedTo,
   #[serde(deserialize_with = "deserialize_one_or_many")]
   pub(crate) to: Vec<Url>,
-  pub(crate) name: String,
+  // If there is inReplyTo field this is actually a comment and must not be parsed
+  #[serde(deserialize_with = "deserialize_not_present", default)]
+  pub(crate) in_reply_to: Option<String>,
 
+  pub(crate) name: Option<String>,
   #[serde(deserialize_with = "deserialize_one_or_many", default)]
   pub(crate) cc: Vec<Url>,
   pub(crate) content: Option<String>,
@@ -259,3 +262,25 @@ impl InCommunity for Page {
     }
   }
 }
+
+/// Only allows deserialization if the field is missing or null. If it is present, throws an error.
+pub fn deserialize_not_present<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
+where
+  D: Deserializer<'de>,
+{
+  let result: Option<String> = Deserialize::deserialize(deserializer)?;
+  match result {
+    None => Ok(None),
+    Some(_) => Err(D::Error::custom("Post must not have inReplyTo property")),
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::protocol::{objects::page::Page, tests::test_parse_lemmy_item};
+
+  #[test]
+  fn test_not_parsing_note_as_page() {
+    assert!(test_parse_lemmy_item::<Page>("assets/lemmy/objects/note.json").is_err());
+  }
+}

scripts/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ex
+set -e
 
 PACKAGE="$1"
 echo "$PACKAGE"