Merge branch 'main' into isomorphic_docker_main_merge_2

Cargo.lock

@@ -325,9 +325,9 @@ dependencies = [
 
 [[package]]
 name = "actix-web"
-version = "3.0.0"
+version = "3.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e665de333edabd0421799822dac3e7d8a25a63bb995ae1f60cd99619d8ddda8"
+checksum = "cd7fc56022da91a4dc00ccae7d7bb82e539749ca36df181695f4efdf5d413b2e"
 dependencies = [
  "actix-codec 0.3.0",
  "actix-http",
@@ -865,9 +865,9 @@ checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd"
 
 [[package]]
 name = "comrak"
-version = "0.7.0"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e17058cc536cf290563e88787d7b9e6030ce4742943017cc2ffb71f88034021c"
+checksum = "0d325e4f2ffff52ca77d995bb675494d5364aa332499d5f7c7fbb28c25e671f6"
 dependencies = [
  "clap",
  "entities",
@@ -875,9 +875,11 @@ dependencies = [
  "pest",
  "pest_derive",
  "regex",
+ "shell-words",
  "twoway",
  "typed-arena",
  "unicode_categories",
+ "xdg",
 ]
 
 [[package]]
@@ -1597,12 +1599,12 @@ dependencies = [
 
 [[package]]
 name = "http-signature-normalization-actix"
-version = "0.4.0-alpha.3"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b44149de8286e9a07aeb72f4dee198530c0fb95df77f36b11138a748788f5603"
+checksum = "88c2cc504dd6a2af53b5f2c3dba63aa5d797359df5b64a21e2230bf78146a373"
 dependencies = [
- "actix-http",
  "actix-web",
+ "awc",
  "base64 0.12.3",
  "bytes",
  "chrono",
@@ -1822,7 +1824,11 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 name = "lemmy_api_structs"
 version = "0.1.0"
 dependencies = [
+ "actix-web",
+ "diesel",
  "lemmy_db",
+ "lemmy_utils",
+ "log",
  "serde 1.0.115",
 ]
 
@@ -3107,6 +3113,12 @@ dependencies = [
  "opaque-debug 0.3.0",
 ]
 
+[[package]]
+name = "shell-words"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074"
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.2.1"
@@ -3236,15 +3248,15 @@ checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
 
 [[package]]
 name = "strum"
-version = "0.18.0"
+version = "0.19.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57bd81eb48f4c437cadc685403cad539345bf703d78e63707418431cecd4522b"
+checksum = "3924a58d165da3b7b2922c667ab0673c7b5fd52b5c19ea3442747bcb3cd15abe"
 
 [[package]]
 name = "strum_macros"
-version = "0.18.0"
+version = "0.19.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87c85aa3f8ea653bfd3ddf25f7ee357ee4d204731f6aa9ad04002306f6e2774c"
+checksum = "2d2ab682ecdcae7f5f45ae85cd7c1e6c8e68ea42c8a612d47fedf831c037146a"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -3914,3 +3926,9 @@ dependencies = [
  "winapi 0.2.8",
  "winapi-build",
 ]
+
+[[package]]
+name = "xdg"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57"

Cargo.toml

@@ -22,7 +22,7 @@ lemmy_rate_limit = { path = "./lemmy_rate_limit" }
 diesel = "1.4.4"
 diesel_migrations = "1.4.0"
 dotenv = "0.15.0"
-activitystreams = "0.7.0-alpha.3"
+activitystreams = "0.7.0-alpha.4"
 activitystreams-ext = "0.1.0-alpha.2"
 bcrypt = "0.8.0"
 chrono = { version = "0.4.7", features = ["serde"] }
@@ -30,15 +30,15 @@ serde_json = { version = "1.0.52", features = ["preserve_order"]}
 serde = { version = "1.0.105", features = ["derive"] }
 actix = "0.10.0"
 actix-web = { version = "3.0.0", default-features = false, features = ["rustls"] }
-actix-files = "0.3.0"
+actix-files = { version = "0.3.0", default-features = false }
-actix-web-actors = "3.0.0"
+actix-web-actors = { version = "3.0.0", default-features = false }
-actix-rt = "1.1.1"
+actix-rt = { version = "1.1.1", default-features = false }
-awc = "2.0.0-alpha.2"
+awc = { version = "2.0.0", default-features = false }
 log = "0.4.0"
 env_logger = "0.7.1"
 rand = "0.7.3"
-strum = "0.18.0"
+strum = "0.19.2"
-strum_macros = "0.18.0"
+strum_macros = "0.19.2"
 jsonwebtoken = "7.0.1"
 lazy_static = "1.3.0"
 rss = "1.9.0"
@@ -46,14 +46,14 @@ url = { version = "2.1.1", features = ["serde"] }
 percent-encoding = "2.1.0"
 openssl = "0.10"
 http = "0.2.1"
-http-signature-normalization-actix = { version = "0.4.0-alpha.2", default-features = false, features = ["sha-2"] }
+http-signature-normalization-actix = { version = "0.4.0", default-features = false, features = ["sha-2"] }
 base64 = "0.12.1"
 tokio = "0.2.21"
 futures = "0.3.5"
 itertools = "0.9.0"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 sha2 = "0.9"
-async-trait = "0.1.36"
+async-trait = "0.1.40"
 captcha = "0.0.7"
 anyhow = "1.0.32"
 thiserror = "1.0.20"

lemmy_api_structs/Cargo.toml

@@ -10,4 +10,8 @@ path = "src/lib.rs"
 
 [dependencies]
 lemmy_db = { path = "../lemmy_db" }
+lemmy_utils = { path = "../lemmy_utils" }
 serde = { version = "1.0.105", features = ["derive"] }
+log = "0.4.0"
+diesel = "1.4.4"
+actix-web = { version = "3.0.0-beta.3", features = ["rustls"] }

lemmy_api_structs/src/lib.rs

@@ -1,3 +1,6 @@
+extern crate actix_web;
+extern crate diesel;
+extern crate log;
 extern crate serde;
 
 pub mod comment;
@@ -5,3 +8,152 @@ pub mod community;
 pub mod post;
 pub mod site;
 pub mod user;
+
+use diesel::PgConnection;
+use lemmy_db::{
+  comment::Comment,
+  post::Post,
+  user::User_,
+  user_mention::{UserMention, UserMentionForm},
+  Crud,
+  DbPool,
+};
+use lemmy_utils::{email::send_email, settings::Settings, utils::MentionData, LemmyError};
+use log::error;
+
+pub async fn blocking<F, T>(pool: &DbPool, f: F) -> Result<T, LemmyError>
+where
+  F: FnOnce(&diesel::PgConnection) -> T + Send + 'static,
+  T: Send + 'static,
+{
+  let pool = pool.clone();
+  let res = actix_web::web::block(move || {
+    let conn = pool.get()?;
+    let res = (f)(&conn);
+    Ok(res) as Result<_, LemmyError>
+  })
+  .await?;
+
+  Ok(res)
+}
+
+pub async fn send_local_notifs(
+  mentions: Vec<MentionData>,
+  comment: Comment,
+  user: &User_,
+  post: Post,
+  pool: &DbPool,
+  do_send_email: bool,
+) -> Result<Vec<i32>, LemmyError> {
+  let user2 = user.clone();
+  let ids = blocking(pool, move |conn| {
+    do_send_local_notifs(conn, &mentions, &comment, &user2, &post, do_send_email)
+  })
+  .await?;
+
+  Ok(ids)
+}
+
+fn do_send_local_notifs(
+  conn: &PgConnection,
+  mentions: &[MentionData],
+  comment: &Comment,
+  user: &User_,
+  post: &Post,
+  do_send_email: bool,
+) -> Vec<i32> {
+  let mut recipient_ids = Vec::new();
+  let hostname = &format!("https://{}", Settings::get().hostname);
+
+  // Send the local mentions
+  for mention in mentions
+    .iter()
+    .filter(|m| m.is_local() && m.name.ne(&user.name))
+    .collect::<Vec<&MentionData>>()
+  {
+    if let Ok(mention_user) = User_::read_from_name(&conn, &mention.name) {
+      // TODO
+      // At some point, make it so you can't tag the parent creator either
+      // This can cause two notifications, one for reply and the other for mention
+      recipient_ids.push(mention_user.id);
+
+      let user_mention_form = UserMentionForm {
+        recipient_id: mention_user.id,
+        comment_id: comment.id,
+        read: None,
+      };
+
+      // Allow this to fail softly, since comment edits might re-update or replace it
+      // Let the uniqueness handle this fail
+      match UserMention::create(&conn, &user_mention_form) {
+        Ok(_mention) => (),
+        Err(_e) => error!("{}", &_e),
+      };
+
+      // Send an email to those users that have notifications on
+      if do_send_email && mention_user.send_notifications_to_email {
+        if let Some(mention_email) = mention_user.email {
+          let subject = &format!("{} - Mentioned by {}", Settings::get().hostname, user.name,);
+          let html = &format!(
+            "<h1>User Mention</h1><br><div>{} - {}</div><br><a href={}/inbox>inbox</a>",
+            user.name, comment.content, hostname
+          );
+          match send_email(subject, &mention_email, &mention_user.name, html) {
+            Ok(_o) => _o,
+            Err(e) => error!("{}", e),
+          };
+        }
+      }
+    }
+  }
+
+  // Send notifs to the parent commenter / poster
+  match comment.parent_id {
+    Some(parent_id) => {
+      if let Ok(parent_comment) = Comment::read(&conn, parent_id) {
+        if parent_comment.creator_id != user.id {
+          if let Ok(parent_user) = User_::read(&conn, parent_comment.creator_id) {
+            recipient_ids.push(parent_user.id);
+
+            if do_send_email && parent_user.send_notifications_to_email {
+              if let Some(comment_reply_email) = parent_user.email {
+                let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,);
+                let html = &format!(
+                  "<h1>Comment Reply</h1><br><div>{} - {}</div><br><a href={}/inbox>inbox</a>",
+                  user.name, comment.content, hostname
+                );
+                match send_email(subject, &comment_reply_email, &parent_user.name, html) {
+                  Ok(_o) => _o,
+                  Err(e) => error!("{}", e),
+                };
+              }
+            }
+          }
+        }
+      }
+    }
+    // Its a post
+    None => {
+      if post.creator_id != user.id {
+        if let Ok(parent_user) = User_::read(&conn, post.creator_id) {
+          recipient_ids.push(parent_user.id);
+
+          if do_send_email && parent_user.send_notifications_to_email {
+            if let Some(post_reply_email) = parent_user.email {
+              let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,);
+              let html = &format!(
+                "<h1>Post Reply</h1><br><div>{} - {}</div><br><a href={}/inbox>inbox</a>",
+                user.name, comment.content, hostname
+              );
+              match send_email(subject, &post_reply_email, &parent_user.name, html) {
+                Ok(_o) => _o,
+                Err(e) => error!("{}", e),
+              };
+            }
+          }
+        }
+      }
+    }
+  };
+  recipient_ids
+}

lemmy_db/Cargo.toml

@@ -12,11 +12,11 @@ diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-
 chrono = { version = "0.4.7", features = ["serde"] }
 serde = { version = "1.0.105", features = ["derive"] }
 serde_json = { version = "1.0.52", features = ["preserve_order"]}
-strum = "0.18.0"
+strum = "0.19.2"
-strum_macros = "0.18.0"
+strum_macros = "0.19.2"
 log = "0.4.0"
 sha2 = "0.9"
-bcrypt = "0.8.0"
+bcrypt = "0.8.2"
 url = { version = "2.1.1", features = ["serde"] }
 lazy_static = "1.3.0"
 regex = "1.3.5"

lemmy_db/src/lib.rs

@@ -40,6 +40,8 @@ pub mod user_mention;
 pub mod user_mention_view;
 pub mod user_view;
 
+pub type DbPool = diesel::r2d2::Pool<diesel::r2d2::ConnectionManager<diesel::PgConnection>>;
+
 pub trait Crud<T> {
   fn create(conn: &PgConnection, form: &T) -> Result<Self, Error>
   where

lemmy_rate_limit/Cargo.toml

@@ -11,8 +11,8 @@ path = "src/lib.rs"
 [dependencies]
 lemmy_utils = { path = "../lemmy_utils" }
 tokio = "0.2.21"
-strum = "0.18.0"
+strum = "0.19.2"
-strum_macros = "0.18.0"
+strum_macros = "0.19.2"
 futures = "0.3.5"
-actix-web = { version = "3.0.0-alpha.3", features = ["rustls"] }
+actix-web = { version = "3.0.0", default-features = false, features = ["rustls"] }
 log = "0.4.0"

lemmy_rate_limit/src/lib.rs

@@ -8,8 +8,8 @@ extern crate tokio;
 use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform};
 use futures::future::{ok, Ready};
 use lemmy_utils::{
-  get_ip,
   settings::{RateLimitConfig, Settings},
+  utils::get_ip,
   LemmyError,
 };
 use rate_limiter::{RateLimitType, RateLimiter};

lemmy_utils/Cargo.toml

@@ -12,18 +12,18 @@ path = "src/lib.rs"
 [dependencies]
 regex = "1.3.5"
 config = { version = "0.10.1", default-features = false, features = ["hjson"] }
-chrono = { version = "0.4.7", features = ["serde"] }
+chrono = { version = "0.4.15", features = ["serde"] }
 lettre = "0.9.3"
 lettre_email = "0.9.4"
 log = "0.4.0"
 itertools = "0.9.0"
 rand = "0.7.3"
-serde = { version = "1.0.105", features = ["derive"] }
+serde = { version = "1.0.115", features = ["derive"] }
 serde_json = { version = "1.0.52", features = ["preserve_order"]}
 thiserror = "1.0.20"
-comrak = "0.7"
+comrak = "0.8.2"
 lazy_static = "1.3.0"
 openssl = "0.10"
 url = { version = "2.1.1", features = ["serde"] }
-actix-web = "3.0.0-alpha.3"
+actix-web = {version = "3.0.0", default-features = false }
 anyhow = "1.0.32"

lemmy_utils/src/apub.rs

@@ -0,0 +1,64 @@
+use crate::settings::Settings;
+use openssl::{pkey::PKey, rsa::Rsa};
+use std::io::{Error, ErrorKind};
+use url::Url;
+
+pub struct Keypair {
+  pub private_key: String,
+  pub public_key: String,
+}
+
+/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
+pub fn generate_actor_keypair() -> Result<Keypair, Error> {
+  let rsa = Rsa::generate(2048)?;
+  let pkey = PKey::from_rsa(rsa)?;
+  let public_key = pkey.public_key_to_pem()?;
+  let private_key = pkey.private_key_to_pem_pkcs8()?;
+  let key_to_string = |key| match String::from_utf8(key) {
+    Ok(s) => Ok(s),
+    Err(e) => Err(Error::new(
+      ErrorKind::Other,
+      format!("Failed converting key to string: {}", e),
+    )),
+  };
+  Ok(Keypair {
+    private_key: key_to_string(private_key)?,
+    public_key: key_to_string(public_key)?,
+  })
+}
+
+pub enum EndpointType {
+  Community,
+  User,
+  Post,
+  Comment,
+  PrivateMessage,
+}
+
+pub fn get_apub_protocol_string() -> &'static str {
+  if Settings::get().federation.tls_enabled {
+    "https"
+  } else {
+    "http"
+  }
+}
+
+/// Generates the ActivityPub ID for a given object type and ID.
+pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
+  let point = match endpoint_type {
+    EndpointType::Community => "c",
+    EndpointType::User => "u",
+    EndpointType::Post => "post",
+    EndpointType::Comment => "comment",
+    EndpointType::PrivateMessage => "private_message",
+  };
+
+  Url::parse(&format!(
+    "{}://{}/{}/{}",
+    get_apub_protocol_string(),
+    Settings::get().hostname,
+    point,
+    name
+  ))
+  .unwrap()
+}

lemmy_utils/src/email.rs

@@ -0,0 +1,55 @@
+use crate::settings::Settings;
+use lettre::{
+  smtp::{
+    authentication::{Credentials, Mechanism},
+    extension::ClientId,
+    ConnectionReuseParameters,
+  },
+  ClientSecurity,
+  SmtpClient,
+  Transport,
+};
+use lettre_email::Email;
+
+pub fn send_email(
+  subject: &str,
+  to_email: &str,
+  to_username: &str,
+  html: &str,
+) -> Result<(), String> {
+  let email_config = Settings::get().email.ok_or("no_email_setup")?;
+
+  let email = Email::builder()
+    .to((to_email, to_username))
+    .from(email_config.smtp_from_address.to_owned())
+    .subject(subject)
+    .html(html)
+    .build()
+    .unwrap();
+
+  let mailer = if email_config.use_tls {
+    SmtpClient::new_simple(&email_config.smtp_server).unwrap()
+  } else {
+    SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap()
+  }
+  .hello_name(ClientId::Domain(Settings::get().hostname))
+  .smtp_utf8(true)
+  .authentication_mechanism(Mechanism::Plain)
+  .connection_reuse(ConnectionReuseParameters::ReuseUnlimited);
+  let mailer = if let (Some(login), Some(password)) =
+    (&email_config.smtp_login, &email_config.smtp_password)
+  {
+    mailer.credentials(Credentials::new(login.to_owned(), password.to_owned()))
+  } else {
+    mailer
+  };
+
+  let mut transport = mailer.transport();
+  let result = transport.send(email.into());
+  transport.close();
+
+  match result {
+    Ok(_) => Ok(()),
+    Err(e) => Err(e.to_string()),
+  }
+}

lemmy_utils/src/lib.rs

@@ -12,29 +12,16 @@ extern crate serde_json;
 extern crate thiserror;
 extern crate url;
 
+pub mod apub;
+pub mod email;
 pub mod settings;
+#[cfg(test)]
+mod test;
+pub mod utils;
 
 use crate::settings::Settings;
-use actix_web::dev::ConnectionInfo;
+use regex::Regex;
-use chrono::{DateTime, FixedOffset, Local, NaiveDateTime};
-use itertools::Itertools;
-use lettre::{
-  smtp::{
-    authentication::{Credentials, Mechanism},
-    extension::ClientId,
-    ConnectionReuseParameters,
-  },
-  ClientSecurity,
-  SmtpClient,
-  Transport,
-};
-use lettre_email::Email;
-use openssl::{pkey::PKey, rsa::Rsa};
-use rand::{distributions::Alphanumeric, thread_rng, Rng};
-use regex::{Regex, RegexBuilder};
-use std::io::{Error, ErrorKind};
 use thiserror::Error;
-use url::Url;
 
 pub type ConnectionId = usize;
 pub type PostId = i32;
@@ -90,236 +77,7 @@ impl std::fmt::Display for LemmyError {
 
 impl actix_web::error::ResponseError for LemmyError {}
 
-pub fn naive_from_unix(time: i64) -> NaiveDateTime {
-  NaiveDateTime::from_timestamp(time, 0)
-}
-
-pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
-  let now = Local::now();
-  DateTime::<FixedOffset>::from_utc(datetime, *now.offset())
-}
-
-pub fn remove_slurs(test: &str) -> String {
-  SLUR_REGEX.replace_all(test, "*removed*").to_string()
-}
-
-pub fn slur_check(test: &str) -> Result<(), Vec<&str>> {
-  let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect();
-
-  // Unique
-  matches.sort_unstable();
-  matches.dedup();
-
-  if matches.is_empty() {
-    Ok(())
-  } else {
-    Err(matches)
-  }
-}
-
-pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String {
-  let start = "No slurs - ";
-  let combined = &slurs.join(", ");
-  [start, combined].concat()
-}
-
-pub fn generate_random_string() -> String {
-  thread_rng().sample_iter(&Alphanumeric).take(30).collect()
-}
-
-pub fn send_email(
-  subject: &str,
-  to_email: &str,
-  to_username: &str,
-  html: &str,
-) -> Result<(), String> {
-  let email_config = Settings::get().email.ok_or("no_email_setup")?;
-
-  let email = Email::builder()
-    .to((to_email, to_username))
-    .from(email_config.smtp_from_address.to_owned())
-    .subject(subject)
-    .html(html)
-    .build()
-    .unwrap();
-
-  let mailer = if email_config.use_tls {
-    SmtpClient::new_simple(&email_config.smtp_server).unwrap()
-  } else {
-    SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap()
-  }
-  .hello_name(ClientId::Domain(Settings::get().hostname))
-  .smtp_utf8(true)
-  .authentication_mechanism(Mechanism::Plain)
-  .connection_reuse(ConnectionReuseParameters::ReuseUnlimited);
-  let mailer = if let (Some(login), Some(password)) =
-    (&email_config.smtp_login, &email_config.smtp_password)
-  {
-    mailer.credentials(Credentials::new(login.to_owned(), password.to_owned()))
-  } else {
-    mailer
-  };
-
-  let mut transport = mailer.transport();
-  let result = transport.send(email.into());
-  transport.close();
-
-  match result {
-    Ok(_) => Ok(()),
-    Err(e) => Err(e.to_string()),
-  }
-}
-
-pub fn markdown_to_html(text: &str) -> String {
-  comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
-}
-
-// TODO nothing is done with community / group webfingers yet, so just ignore those for now
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub struct MentionData {
-  pub name: String,
-  pub domain: String,
-}
-
-impl MentionData {
-  pub fn is_local(&self) -> bool {
-    Settings::get().hostname.eq(&self.domain)
-  }
-  pub fn full_name(&self) -> String {
-    format!("@{}@{}", &self.name, &self.domain)
-  }
-}
-
-pub fn scrape_text_for_mentions(text: &str) -> Vec<MentionData> {
-  let mut out: Vec<MentionData> = Vec::new();
-  for caps in MENTIONS_REGEX.captures_iter(text) {
-    out.push(MentionData {
-      name: caps["name"].to_string(),
-      domain: caps["domain"].to_string(),
-    });
-  }
-  out.into_iter().unique().collect()
-}
-
-pub fn is_valid_username(name: &str) -> bool {
-  VALID_USERNAME_REGEX.is_match(name)
-}
-
-// Can't do a regex here, reverse lookarounds not supported
-pub fn is_valid_preferred_username(preferred_username: &str) -> bool {
-  !preferred_username.starts_with('@')
-    && preferred_username.len() >= 3
-    && preferred_username.len() <= 20
-}
-
-pub fn is_valid_community_name(name: &str) -> bool {
-  VALID_COMMUNITY_NAME_REGEX.is_match(name)
-}
-
-pub fn is_valid_post_title(title: &str) -> bool {
-  VALID_POST_TITLE_REGEX.is_match(title)
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::{
-    is_valid_community_name,
-    is_valid_post_title,
-    is_valid_preferred_username,
-    is_valid_username,
-    remove_slurs,
-    scrape_text_for_mentions,
-    slur_check,
-    slurs_vec_to_str,
-  };
-
-  #[test]
-  fn test_mentions_regex() {
-    let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)";
-    let mentions = scrape_text_for_mentions(text);
-
-    assert_eq!(mentions[0].name, "tedu".to_string());
-    assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string());
-    assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string());
-  }
-
-  #[test]
-  fn test_valid_register_username() {
-    assert!(is_valid_username("Hello_98"));
-    assert!(is_valid_username("ten"));
-    assert!(!is_valid_username("Hello-98"));
-    assert!(!is_valid_username("a"));
-    assert!(!is_valid_username(""));
-  }
-
-  #[test]
-  fn test_valid_preferred_username() {
-    assert!(is_valid_preferred_username("hello @there"));
-    assert!(!is_valid_preferred_username("@hello there"));
-  }
-
-  #[test]
-  fn test_valid_community_name() {
-    assert!(is_valid_community_name("example"));
-    assert!(is_valid_community_name("example_community"));
-    assert!(!is_valid_community_name("Example"));
-    assert!(!is_valid_community_name("Ex"));
-    assert!(!is_valid_community_name(""));
-  }
-
-  #[test]
-  fn test_valid_post_title() {
-    assert!(is_valid_post_title("Post Title"));
-    assert!(is_valid_post_title("   POST TITLE 😃😃😃😃😃"));
-    assert!(!is_valid_post_title("\n \n \n \n    		")); // tabs/spaces/newlines
-  }
-
-  #[test]
-  fn test_slur_filter() {
-    let test =
-      "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
-    let slur_free = "No slurs here";
-    assert_eq!(
-      remove_slurs(&test),
-      "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
-        .to_string()
-    );
-
-    let has_slurs_vec = vec![
-      "Niggerz",
-      "coons",
-      "dindu",
-      "ladyboy",
-      "retardeds",
-      "tranny",
-    ];
-    let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny";
-
-    assert_eq!(slur_check(test), Err(has_slurs_vec));
-    assert_eq!(slur_check(slur_free), Ok(()));
-    if let Err(slur_vec) = slur_check(test) {
-      assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str);
-    }
-  }
-
-  // These helped with testing
-  // #[test]
-  // fn test_send_email() {
-  //  let result =  send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
-  //   assert!(result.is_ok());
-  // }
-}
-
 lazy_static! {
-  static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap();
-  static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap();
-  static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap();
-  // TODO keep this old one, it didn't work with port well tho
-  // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap();
-  static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._:-]+)").unwrap();
-  static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap();
-  static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap();
-  static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap();
   pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!(
     "^group:([a-z0-9_]{{3, 20}})@{}$",
     Settings::get().hostname
@@ -333,73 +91,3 @@ lazy_static! {
   pub static ref CACHE_CONTROL_REGEX: Regex =
     Regex::new("^((text|image)/.+|application/javascript)$").unwrap();
 }
-
-pub struct Keypair {
-  pub private_key: String,
-  pub public_key: String,
-}
-
-/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
-pub fn generate_actor_keypair() -> Result<Keypair, Error> {
-  let rsa = Rsa::generate(2048)?;
-  let pkey = PKey::from_rsa(rsa)?;
-  let public_key = pkey.public_key_to_pem()?;
-  let private_key = pkey.private_key_to_pem_pkcs8()?;
-  let key_to_string = |key| match String::from_utf8(key) {
-    Ok(s) => Ok(s),
-    Err(e) => Err(Error::new(
-      ErrorKind::Other,
-      format!("Failed converting key to string: {}", e),
-    )),
-  };
-  Ok(Keypair {
-    private_key: key_to_string(private_key)?,
-    public_key: key_to_string(public_key)?,
-  })
-}
-
-pub enum EndpointType {
-  Community,
-  User,
-  Post,
-  Comment,
-  PrivateMessage,
-}
-
-pub fn get_apub_protocol_string() -> &'static str {
-  if Settings::get().federation.tls_enabled {
-    "https"
-  } else {
-    "http"
-  }
-}
-
-/// Generates the ActivityPub ID for a given object type and ID.
-pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
-  let point = match endpoint_type {
-    EndpointType::Community => "c",
-    EndpointType::User => "u",
-    EndpointType::Post => "post",
-    EndpointType::Comment => "comment",
-    EndpointType::PrivateMessage => "private_message",
-  };
-
-  Url::parse(&format!(
-    "{}://{}/{}/{}",
-    get_apub_protocol_string(),
-    Settings::get().hostname,
-    point,
-    name
-  ))
-  .unwrap()
-}
-
-pub fn get_ip(conn_info: &ConnectionInfo) -> String {
-  conn_info
-    .realip_remote_addr()
-    .unwrap_or("127.0.0.1:12345")
-    .split(':')
-    .next()
-    .unwrap_or("127.0.0.1")
-    .to_string()
-}

lemmy_utils/src/test.rs

@@ -0,0 +1,86 @@
+use crate::utils::{
+  is_valid_community_name,
+  is_valid_post_title,
+  is_valid_preferred_username,
+  is_valid_username,
+  remove_slurs,
+  scrape_text_for_mentions,
+  slur_check,
+  slurs_vec_to_str,
+};
+
+#[test]
+fn test_mentions_regex() {
+  let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)";
+  let mentions = scrape_text_for_mentions(text);
+
+  assert_eq!(mentions[0].name, "tedu".to_string());
+  assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string());
+  assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string());
+}
+
+#[test]
+fn test_valid_register_username() {
+  assert!(is_valid_username("Hello_98"));
+  assert!(is_valid_username("ten"));
+  assert!(!is_valid_username("Hello-98"));
+  assert!(!is_valid_username("a"));
+  assert!(!is_valid_username(""));
+}
+
+#[test]
+fn test_valid_preferred_username() {
+  assert!(is_valid_preferred_username("hello @there"));
+  assert!(!is_valid_preferred_username("@hello there"));
+}
+
+#[test]
+fn test_valid_community_name() {
+  assert!(is_valid_community_name("example"));
+  assert!(is_valid_community_name("example_community"));
+  assert!(!is_valid_community_name("Example"));
+  assert!(!is_valid_community_name("Ex"));
+  assert!(!is_valid_community_name(""));
+}
+
+#[test]
+fn test_valid_post_title() {
+  assert!(is_valid_post_title("Post Title"));
+  assert!(is_valid_post_title("   POST TITLE 😃😃😃😃😃"));
+  assert!(!is_valid_post_title("\n \n \n \n    		")); // tabs/spaces/newlines
+}
+
+#[test]
+fn test_slur_filter() {
+  let test =
+      "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
+  let slur_free = "No slurs here";
+  assert_eq!(
+      remove_slurs(&test),
+      "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
+        .to_string()
+    );
+
+  let has_slurs_vec = vec![
+    "Niggerz",
+    "coons",
+    "dindu",
+    "ladyboy",
+    "retardeds",
+    "tranny",
+  ];
+  let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny";
+
+  assert_eq!(slur_check(test), Err(has_slurs_vec));
+  assert_eq!(slur_check(slur_free), Ok(()));
+  if let Err(slur_vec) = slur_check(test) {
+    assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str);
+  }
+}
+
+// These helped with testing
+// #[test]
+// fn test_send_email() {
+//  let result =  send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
+//   assert!(result.is_ok());
+// }

lemmy_utils/src/utils.rs

@@ -0,0 +1,130 @@
+use crate::{settings::Settings, APIError};
+use actix_web::dev::ConnectionInfo;
+use chrono::{DateTime, FixedOffset, Local, NaiveDateTime};
+use itertools::Itertools;
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use regex::{Regex, RegexBuilder};
+
+lazy_static! {
+static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap();
+static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap();
+static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap();
+// TODO keep this old one, it didn't work with port well tho
+// static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap();
+static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._:-]+)").unwrap();
+static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap();
+static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap();
+static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap();
+}
+
+pub fn naive_from_unix(time: i64) -> NaiveDateTime {
+  NaiveDateTime::from_timestamp(time, 0)
+}
+
+pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
+  let now = Local::now();
+  DateTime::<FixedOffset>::from_utc(datetime, *now.offset())
+}
+
+pub fn remove_slurs(test: &str) -> String {
+  SLUR_REGEX.replace_all(test, "*removed*").to_string()
+}
+
+pub(crate) fn slur_check(test: &str) -> Result<(), Vec<&str>> {
+  let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect();
+
+  // Unique
+  matches.sort_unstable();
+  matches.dedup();
+
+  if matches.is_empty() {
+    Ok(())
+  } else {
+    Err(matches)
+  }
+}
+
+pub fn check_slurs(text: &str) -> Result<(), APIError> {
+  if let Err(slurs) = slur_check(text) {
+    Err(APIError::err(&slurs_vec_to_str(slurs)))
+  } else {
+    Ok(())
+  }
+}
+
+pub fn check_slurs_opt(text: &Option<String>) -> Result<(), APIError> {
+  match text {
+    Some(t) => check_slurs(t),
+    None => Ok(()),
+  }
+}
+
+pub(crate) fn slurs_vec_to_str(slurs: Vec<&str>) -> String {
+  let start = "No slurs - ";
+  let combined = &slurs.join(", ");
+  [start, combined].concat()
+}
+
+pub fn generate_random_string() -> String {
+  thread_rng().sample_iter(&Alphanumeric).take(30).collect()
+}
+
+pub fn markdown_to_html(text: &str) -> String {
+  comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
+}
+
+// TODO nothing is done with community / group webfingers yet, so just ignore those for now
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct MentionData {
+  pub name: String,
+  pub domain: String,
+}
+
+impl MentionData {
+  pub fn is_local(&self) -> bool {
+    Settings::get().hostname.eq(&self.domain)
+  }
+  pub fn full_name(&self) -> String {
+    format!("@{}@{}", &self.name, &self.domain)
+  }
+}
+
+pub fn scrape_text_for_mentions(text: &str) -> Vec<MentionData> {
+  let mut out: Vec<MentionData> = Vec::new();
+  for caps in MENTIONS_REGEX.captures_iter(text) {
+    out.push(MentionData {
+      name: caps["name"].to_string(),
+      domain: caps["domain"].to_string(),
+    });
+  }
+  out.into_iter().unique().collect()
+}
+
+pub fn is_valid_username(name: &str) -> bool {
+  VALID_USERNAME_REGEX.is_match(name)
+}
+
+// Can't do a regex here, reverse lookarounds not supported
+pub fn is_valid_preferred_username(preferred_username: &str) -> bool {
+  !preferred_username.starts_with('@')
+    && preferred_username.len() >= 3
+    && preferred_username.len() <= 20
+}
+
+pub fn is_valid_community_name(name: &str) -> bool {
+  VALID_COMMUNITY_NAME_REGEX.is_match(name)
+}
+
+pub fn is_valid_post_title(title: &str) -> bool {
+  VALID_POST_TITLE_REGEX.is_match(title)
+}
+
+pub fn get_ip(conn_info: &ConnectionInfo) -> String {
+  conn_info
+    .realip_remote_addr()
+    .unwrap_or("127.0.0.1:12345")
+    .split(':')
+    .next()
+    .unwrap_or("127.0.0.1")
+    .to_string()
+}

src/api/comment.rs

@@ -8,13 +8,11 @@ use crate::{
     Perform,
   },
   apub::{ApubLikeableType, ApubObjectType},
-  blocking,
   websocket::{messages::SendComment, UserOperation},
-  DbPool,
   LemmyContext,
 };
 use actix_web::web::Data;
-use lemmy_api_structs::comment::*;
+use lemmy_api_structs::{blocking, comment::*, send_local_notifs};
 use lemmy_db::{
   comment::*,
   comment_view::*,
@@ -22,7 +20,6 @@ use lemmy_db::{
   post::*,
   site_view::*,
   user::*,
-  user_mention::*,
   Crud,
   Likeable,
   ListingType,
@@ -30,18 +27,12 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  make_apub_endpoint,
+  apub::{make_apub_endpoint, EndpointType},
-  remove_slurs,
+  utils::{remove_slurs, scrape_text_for_mentions},
-  scrape_text_for_mentions,
-  send_email,
-  settings::Settings,
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
-  MentionData,
 };
-use log::error;
 use std::str::FromStr;
 
 #[async_trait::async_trait(?Send)]
@@ -694,124 +685,3 @@ impl Perform for GetComments {
     Ok(GetCommentsResponse { comments })
   }
 }
-
-pub async fn send_local_notifs(
-  mentions: Vec<MentionData>,
-  comment: Comment,
-  user: &User_,
-  post: Post,
-  pool: &DbPool,
-  do_send_email: bool,
-) -> Result<Vec<i32>, LemmyError> {
-  let user2 = user.clone();
-  let ids = blocking(pool, move |conn| {
-    do_send_local_notifs(conn, &mentions, &comment, &user2, &post, do_send_email)
-  })
-  .await?;
-
-  Ok(ids)
-}
-
-fn do_send_local_notifs(
-  conn: &diesel::PgConnection,
-  mentions: &[MentionData],
-  comment: &Comment,
-  user: &User_,
-  post: &Post,
-  do_send_email: bool,
-) -> Vec<i32> {
-  let mut recipient_ids = Vec::new();
-  let hostname = &format!("https://{}", Settings::get().hostname);
-
-  // Send the local mentions
-  for mention in mentions
-    .iter()
-    .filter(|m| m.is_local() && m.name.ne(&user.name))
-    .collect::<Vec<&MentionData>>()
-  {
-    if let Ok(mention_user) = User_::read_from_name(&conn, &mention.name) {
-      // TODO
-      // At some point, make it so you can't tag the parent creator either
-      // This can cause two notifications, one for reply and the other for mention
-      recipient_ids.push(mention_user.id);
-
-      let user_mention_form = UserMentionForm {
-        recipient_id: mention_user.id,
-        comment_id: comment.id,
-        read: None,
-      };
-
-      // Allow this to fail softly, since comment edits might re-update or replace it
-      // Let the uniqueness handle this fail
-      match UserMention::create(&conn, &user_mention_form) {
-        Ok(_mention) => (),
-        Err(_e) => error!("{}", &_e),
-      };
-
-      // Send an email to those users that have notifications on
-      if do_send_email && mention_user.send_notifications_to_email {
-        if let Some(mention_email) = mention_user.email {
-          let subject = &format!("{} - Mentioned by {}", Settings::get().hostname, user.name,);
-          let html = &format!(
-            "<h1>User Mention</h1><br><div>{} - {}</div><br><a href={}/inbox>inbox</a>",
-            user.name, comment.content, hostname
-          );
-          match send_email(subject, &mention_email, &mention_user.name, html) {
-            Ok(_o) => _o,
-            Err(e) => error!("{}", e),
-          };
-        }
-      }
-    }
-  }
-
-  // Send notifs to the parent commenter / poster
-  match comment.parent_id {
-    Some(parent_id) => {
-      if let Ok(parent_comment) = Comment::read(&conn, parent_id) {
-        if parent_comment.creator_id != user.id {
-          if let Ok(parent_user) = User_::read(&conn, parent_comment.creator_id) {
-            recipient_ids.push(parent_user.id);
-
-            if do_send_email && parent_user.send_notifications_to_email {
-              if let Some(comment_reply_email) = parent_user.email {
-                let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,);
-                let html = &format!(
-                  "<h1>Comment Reply</h1><br><div>{} - {}</div><br><a href={}/inbox>inbox</a>",
-                  user.name, comment.content, hostname
-                );
-                match send_email(subject, &comment_reply_email, &parent_user.name, html) {
-                  Ok(_o) => _o,
-                  Err(e) => error!("{}", e),
-                };
-              }
-            }
-          }
-        }
-      }
-    }
-    // Its a post
-    None => {
-      if post.creator_id != user.id {
-        if let Ok(parent_user) = User_::read(&conn, post.creator_id) {
-          recipient_ids.push(parent_user.id);
-
-          if do_send_email && parent_user.send_notifications_to_email {
-            if let Some(post_reply_email) = parent_user.email {
-              let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,);
-              let html = &format!(
-                "<h1>Post Reply</h1><br><div>{} - {}</div><br><a href={}/inbox>inbox</a>",
-                user.name, comment.content, hostname
-              );
-              match send_email(subject, &post_reply_email, &parent_user.name, html) {
-                Ok(_o) => _o,
-                Err(e) => error!("{}", e),
-              };
-            }
-          }
-        }
-      }
-    }
-  };
-  recipient_ids
-}

src/api/community.rs

@@ -1,15 +1,6 @@
 use crate::{
-  api::{
+  api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, is_mod_or_admin, Perform},
-    check_slurs,
-    check_slurs_opt,
-    get_user_from_jwt,
-    get_user_from_jwt_opt,
-    is_admin,
-    is_mod_or_admin,
-    Perform,
-  },
   apub::ActorType,
-  blocking,
   websocket::{
     messages::{GetCommunityUsersOnline, JoinCommunityRoom, SendCommunityRoomMessage},
     UserOperation,
@@ -18,7 +9,7 @@ use crate::{
 };
 use actix_web::web::Data;
 use anyhow::Context;
-use lemmy_api_structs::community::*;
+use lemmy_api_structs::{blocking, community::*};
 use lemmy_db::{
   comment::Comment,
   comment_view::CommentQueryBuilder,
@@ -37,14 +28,11 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  generate_actor_keypair,
+  apub::{generate_actor_keypair, make_apub_endpoint, EndpointType},
-  is_valid_community_name,
   location_info,
-  make_apub_endpoint,
+  utils::{check_slurs, check_slurs_opt, is_valid_community_name, naive_from_unix},
-  naive_from_unix,
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
 };
 use std::str::FromStr;

src/api/mod.rs

@@ -1,5 +1,6 @@
-use crate::{api::claims::Claims, blocking, DbPool, LemmyContext};
+use crate::{api::claims::Claims, DbPool, LemmyContext};
 use actix_web::web::Data;
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   community::Community,
   community_view::CommunityUserBanView,
@@ -7,7 +8,7 @@ use lemmy_db::{
   user::User_,
   Crud,
 };
-use lemmy_utils::{slur_check, slurs_vec_to_str, APIError, ConnectionId, LemmyError};
+use lemmy_utils::{APIError, ConnectionId, LemmyError};
 
 pub mod claims;
 pub mod comment;
@@ -83,19 +84,6 @@ pub(in crate::api) async fn get_user_from_jwt_opt(
   }
 }
 
-pub(in crate) fn check_slurs(text: &str) -> Result<(), APIError> {
-  if let Err(slurs) = slur_check(text) {
-    Err(APIError::err(&slurs_vec_to_str(slurs)))
-  } else {
-    Ok(())
-  }
-}
-pub(in crate) fn check_slurs_opt(text: &Option<String>) -> Result<(), APIError> {
-  match text {
-    Some(t) => check_slurs(t),
-    None => Ok(()),
-  }
-}
 pub(in crate::api) async fn check_community_ban(
   user_id: i32,
   community_id: i32,

src/api/post.rs

@@ -1,15 +1,6 @@
 use crate::{
-  api::{
+  api::{check_community_ban, get_user_from_jwt, get_user_from_jwt_opt, is_mod_or_admin, Perform},
-    check_community_ban,
-    check_slurs,
-    check_slurs_opt,
-    get_user_from_jwt,
-    get_user_from_jwt_opt,
-    is_mod_or_admin,
-    Perform,
-  },
   apub::{ApubLikeableType, ApubObjectType},
-  blocking,
   fetch_iframely_and_pictrs_data,
   websocket::{
     messages::{GetPostUsersOnline, JoinPostRoom, SendPost},
@@ -18,7 +9,7 @@ use crate::{
   LemmyContext,
 };
 use actix_web::web::Data;
-use lemmy_api_structs::post::*;
+use lemmy_api_structs::{blocking, post::*};
 use lemmy_db::{
   comment_view::*,
   community_view::*,
@@ -34,11 +25,10 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  is_valid_post_title,
+  apub::{make_apub_endpoint, EndpointType},
-  make_apub_endpoint,
+  utils::{check_slurs, check_slurs_opt, is_valid_post_title},
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
 };
 use std::str::FromStr;

src/api/site.rs

@@ -1,14 +1,6 @@
 use crate::{
-  api::{
+  api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
-    check_slurs,
-    check_slurs_opt,
-    get_user_from_jwt,
-    get_user_from_jwt_opt,
-    is_admin,
-    Perform,
-  },
   apub::fetcher::search_by_apub_id,
-  blocking,
   version,
   websocket::{
     messages::{GetUsersOnline, SendAllMessage},
@@ -18,7 +10,7 @@ use crate::{
 };
 use actix_web::web::Data;
 use anyhow::Context;
-use lemmy_api_structs::{site::*, user::Register};
+use lemmy_api_structs::{blocking, site::*, user::Register};
 use lemmy_db::{
   category::*,
   comment_view::*,
@@ -35,7 +27,14 @@ use lemmy_db::{
   SearchType,
   SortType,
 };
-use lemmy_utils::{location_info, settings::Settings, APIError, ConnectionId, LemmyError};
+use lemmy_utils::{
+  location_info,
+  settings::Settings,
+  utils::{check_slurs, check_slurs_opt},
+  APIError,
+  ConnectionId,
+  LemmyError,
+};
 use log::{debug, info};
 use std::str::FromStr;
 

src/api/user.rs

@@ -1,7 +1,6 @@
 use crate::{
-  api::{check_slurs, claims::Claims, get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
+  api::{claims::Claims, get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
   apub::ApubObjectType,
-  blocking,
   captcha_espeak_wav_base64,
   websocket::{
     messages::{CaptchaItem, CheckCaptcha, JoinUserRoom, SendAllMessage, SendUserRoomMessage},
@@ -14,7 +13,7 @@ use anyhow::Context;
 use bcrypt::verify;
 use captcha::{gen, Difficulty};
 use chrono::Duration;
-use lemmy_api_structs::user::*;
+use lemmy_api_structs::{blocking, user::*};
 use lemmy_db::{
   comment::*,
   comment_view::*,
@@ -41,19 +40,20 @@ use lemmy_db::{
   SortType,
 };
 use lemmy_utils::{
-  generate_actor_keypair,
+  apub::{generate_actor_keypair, make_apub_endpoint, EndpointType},
-  generate_random_string,
+  email::send_email,
-  is_valid_preferred_username,
-  is_valid_username,
   location_info,
-  make_apub_endpoint,
-  naive_from_unix,
-  remove_slurs,
-  send_email,
   settings::Settings,
+  utils::{
+    check_slurs,
+    generate_random_string,
+    is_valid_preferred_username,
+    is_valid_username,
+    naive_from_unix,
+    remove_slurs,
+  },
   APIError,
   ConnectionId,
-  EndpointType,
   LemmyError,
 };
 use log::error;

src/apub/activities.rs

@@ -7,7 +7,7 @@ use activitystreams::{
   object::AsObject,
 };
 use lemmy_db::{community::Community, user::User_};
-use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError};
+use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
 use serde::{export::fmt::Debug, Serialize};
 use url::{ParseError, Url};
 use uuid::Uuid;

src/apub/comment.rs

@@ -17,7 +17,6 @@ use crate::{
     FromApub,
     ToApub,
   },
-  blocking,
   DbPool,
   LemmyContext,
 };
@@ -41,6 +40,7 @@ use activitystreams::{
 use actix_web::{body::Body, web, web::Path, HttpResponse};
 use anyhow::Context;
 use itertools::Itertools;
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   comment::{Comment, CommentForm},
   community::Community,
@@ -49,12 +49,9 @@ use lemmy_db::{
   Crud,
 };
 use lemmy_utils::{
-  convert_datetime,
   location_info,
-  remove_slurs,
+  utils::{convert_datetime, remove_slurs, scrape_text_for_mentions, MentionData},
-  scrape_text_for_mentions,
   LemmyError,
-  MentionData,
 };
 use log::debug;
 use serde::Deserialize;

src/apub/community.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::{check_slurs, check_slurs_opt},
   apub::{
     activities::generate_activity_id,
     activity_queue::send_activity,
@@ -15,7 +14,6 @@ use crate::{
     GroupExt,
     ToApub,
   },
-  blocking,
   DbPool,
   LemmyContext,
 };
@@ -40,6 +38,7 @@ use activitystreams_ext::Ext2;
 use actix_web::{body::Body, web, HttpResponse};
 use anyhow::Context;
 use itertools::Itertools;
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   community::{Community, CommunityForm},
   community_view::{CommunityFollowerView, CommunityModeratorView},
@@ -47,7 +46,12 @@ use lemmy_db::{
   post::Post,
   user::User_,
 };
-use lemmy_utils::{convert_datetime, get_apub_protocol_string, location_info, LemmyError};
+use lemmy_utils::{
+  apub::get_apub_protocol_string,
+  location_info,
+  utils::{check_slurs, check_slurs_opt, convert_datetime},
+  LemmyError,
+};
 use serde::Deserialize;
 use url::Url;
 

src/apub/fetcher.rs

@@ -8,7 +8,6 @@ use crate::{
     PersonExt,
     APUB_JSON_CONTENT_TYPE,
   },
-  blocking,
   request::{retry, RecvError},
   LemmyContext,
 };
@@ -16,7 +15,7 @@ use activitystreams::{base::BaseExt, collection::OrderedCollection, object::Note
 use anyhow::{anyhow, Context};
 use chrono::NaiveDateTime;
 use diesel::result::Error::NotFound;
-use lemmy_api_structs::site::SearchResponse;
+use lemmy_api_structs::{blocking, site::SearchResponse};
 use lemmy_db::{
   comment::{Comment, CommentForm},
   comment_view::CommentView,
@@ -31,7 +30,7 @@ use lemmy_db::{
   Joinable,
   SearchType,
 };
-use lemmy_utils::{get_apub_protocol_string, location_info, LemmyError};
+use lemmy_utils::{apub::get_apub_protocol_string, location_info, LemmyError};
 use log::debug;
 use reqwest::Client;
 use serde::Deserialize;

src/apub/inbox/activities/create.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::comment::send_local_notifs,
   apub::{
     inbox::shared_inbox::{
       announce_if_community_is_local,
@@ -10,7 +9,6 @@ use crate::{
     FromApub,
     PageExt,
   },
-  blocking,
   websocket::{
     messages::{SendComment, SendPost},
     UserOperation,
@@ -20,14 +18,19 @@ use crate::{
 use activitystreams::{activity::Create, base::AnyBase, object::Note, prelude::*};
 use actix_web::HttpResponse;
 use anyhow::Context;
-use lemmy_api_structs::{comment::CommentResponse, post::PostResponse};
+use lemmy_api_structs::{
+  blocking,
+  comment::CommentResponse,
+  post::PostResponse,
+  send_local_notifs,
+};
 use lemmy_db::{
   comment::{Comment, CommentForm},
   comment_view::CommentView,
   post::{Post, PostForm},
   post_view::PostView,
 };
-use lemmy_utils::{location_info, scrape_text_for_mentions, LemmyError};
+use lemmy_utils::{location_info, utils::scrape_text_for_mentions, LemmyError};
 
 pub async fn receive_create(
   activity: AnyBase,

src/apub/inbox/activities/delete.rs

@@ -11,7 +11,6 @@ use crate::{
     GroupExt,
     PageExt,
   },
-  blocking,
   websocket::{
     messages::{SendComment, SendCommunityRoomMessage, SendPost},
     UserOperation,
@@ -22,6 +21,7 @@ use activitystreams::{activity::Delete, base::AnyBase, object::Note, prelude::*}
 use actix_web::HttpResponse;
 use anyhow::Context;
 use lemmy_api_structs::{
+  blocking,
   comment::CommentResponse,
   community::CommunityResponse,
   post::PostResponse,

src/apub/inbox/activities/dislike.rs

@@ -9,7 +9,6 @@ use crate::{
     FromApub,
     PageExt,
   },
-  blocking,
   websocket::{
     messages::{SendComment, SendPost},
     UserOperation,
@@ -19,7 +18,7 @@ use crate::{
 use activitystreams::{activity::Dislike, base::AnyBase, object::Note, prelude::*};
 use actix_web::HttpResponse;
 use anyhow::Context;
-use lemmy_api_structs::{comment::CommentResponse, post::PostResponse};
+use lemmy_api_structs::{blocking, comment::CommentResponse, post::PostResponse};
 use lemmy_db::{
   comment::{CommentForm, CommentLike, CommentLikeForm},
   comment_view::CommentView,

src/apub/inbox/activities/like.rs

@@ -9,7 +9,6 @@ use crate::{
     FromApub,
     PageExt,
   },
-  blocking,
   websocket::{
     messages::{SendComment, SendPost},
     UserOperation,
@@ -19,7 +18,7 @@ use crate::{
 use activitystreams::{activity::Like, base::AnyBase, object::Note, prelude::*};
 use actix_web::HttpResponse;
 use anyhow::Context;
-use lemmy_api_structs::{comment::CommentResponse, post::PostResponse};
+use lemmy_api_structs::{blocking, comment::CommentResponse, post::PostResponse};
 use lemmy_db::{
   comment::{CommentForm, CommentLike, CommentLikeForm},
   comment_view::CommentView,

src/apub/inbox/activities/remove.rs

@@ -12,7 +12,6 @@ use crate::{
     GroupExt,
     PageExt,
   },
-  blocking,
   websocket::{
     messages::{SendComment, SendCommunityRoomMessage, SendPost},
     UserOperation,
@@ -23,6 +22,7 @@ use activitystreams::{activity::Remove, base::AnyBase, object::Note, prelude::*}
 use actix_web::HttpResponse;
 use anyhow::{anyhow, Context};
 use lemmy_api_structs::{
+  blocking,
   comment::CommentResponse,
   community::CommunityResponse,
   post::PostResponse,

src/apub/inbox/activities/undo.rs

@@ -11,7 +11,6 @@ use crate::{
     GroupExt,
     PageExt,
   },
-  blocking,
   websocket::{
     messages::{SendComment, SendCommunityRoomMessage, SendPost},
     UserOperation,
@@ -27,6 +26,7 @@ use activitystreams::{
 use actix_web::HttpResponse;
 use anyhow::{anyhow, Context};
 use lemmy_api_structs::{
+  blocking,
   comment::CommentResponse,
   community::CommunityResponse,
   post::PostResponse,

src/apub/inbox/activities/update.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::comment::send_local_notifs,
   apub::{
     fetcher::{get_or_fetch_and_insert_comment, get_or_fetch_and_insert_post},
     inbox::shared_inbox::{
@@ -11,7 +10,6 @@ use crate::{
     FromApub,
     PageExt,
   },
-  blocking,
   websocket::{
     messages::{SendComment, SendPost},
     UserOperation,
@@ -21,7 +19,12 @@ use crate::{
 use activitystreams::{activity::Update, base::AnyBase, object::Note, prelude::*};
 use actix_web::HttpResponse;
 use anyhow::Context;
-use lemmy_api_structs::{comment::CommentResponse, post::PostResponse};
+use lemmy_api_structs::{
+  blocking,
+  comment::CommentResponse,
+  post::PostResponse,
+  send_local_notifs,
+};
 use lemmy_db::{
   comment::{Comment, CommentForm},
   comment_view::CommentView,
@@ -29,7 +32,7 @@ use lemmy_db::{
   post_view::PostView,
   Crud,
 };
-use lemmy_utils::{location_info, scrape_text_for_mentions, LemmyError};
+use lemmy_utils::{location_info, utils::scrape_text_for_mentions, LemmyError};
 
 pub async fn receive_update(
   activity: AnyBase,

src/apub/inbox/community_inbox.rs

@@ -6,7 +6,6 @@ use crate::{
     insert_activity,
     ActorType,
   },
-  blocking,
   LemmyContext,
 };
 use activitystreams::{
@@ -16,6 +15,7 @@ use activitystreams::{
 };
 use actix_web::{web, HttpRequest, HttpResponse};
 use anyhow::{anyhow, Context};
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   community::{Community, CommunityFollower, CommunityFollowerForm},
   user::User_,

src/apub/inbox/user_inbox.rs

@@ -6,7 +6,6 @@ use crate::{
     insert_activity,
     FromApub,
   },
-  blocking,
   websocket::{messages::SendUserRoomMessage, UserOperation},
   LemmyContext,
 };
@@ -18,7 +17,7 @@ use activitystreams::{
 };
 use actix_web::{web, HttpRequest, HttpResponse};
 use anyhow::Context;
-use lemmy_api_structs::user::PrivateMessageResponse;
+use lemmy_api_structs::{blocking, user::PrivateMessageResponse};
 use lemmy_db::{
   community::{CommunityFollower, CommunityFollowerForm},
   naive_now,

src/apub/mod.rs

@@ -15,7 +15,6 @@ use crate::{
     page_extension::PageExtension,
     signatures::{PublicKey, PublicKeyExtension},
   },
-  blocking,
   request::{retry, RecvError},
   routes::webfinger::WebFingerResponse,
   DbPool,
@@ -33,14 +32,14 @@ use activitystreams_ext::{Ext1, Ext2};
 use actix_web::{body::Body, HttpResponse};
 use anyhow::{anyhow, Context};
 use chrono::NaiveDateTime;
+use lemmy_api_structs::blocking;
 use lemmy_db::{activity::do_insert_activity, user::User_};
 use lemmy_utils::{
-  convert_datetime,
+  apub::get_apub_protocol_string,
-  get_apub_protocol_string,
   location_info,
   settings::Settings,
+  utils::{convert_datetime, MentionData},
   LemmyError,
-  MentionData,
 };
 use log::debug;
 use reqwest::Client;

src/apub/post.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::check_slurs,
   apub::{
     activities::{generate_activity_id, send_activity_to_community},
     check_actor_domain,
@@ -15,7 +14,6 @@ use crate::{
     PageExt,
     ToApub,
   },
-  blocking,
   DbPool,
   LemmyContext,
 };
@@ -37,13 +35,18 @@ use activitystreams::{
 use activitystreams_ext::Ext1;
 use actix_web::{body::Body, web, HttpResponse};
 use anyhow::Context;
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   community::Community,
   post::{Post, PostForm},
   user::User_,
   Crud,
 };
-use lemmy_utils::{convert_datetime, location_info, remove_slurs, LemmyError};
+use lemmy_utils::{
+  location_info,
+  utils::{check_slurs, convert_datetime, remove_slurs},
+  LemmyError,
+};
 use serde::Deserialize;
 use url::Url;
 

src/apub/private_message.rs

@@ -12,7 +12,6 @@ use crate::{
     FromApub,
     ToApub,
   },
-  blocking,
   DbPool,
   LemmyContext,
 };
@@ -28,12 +27,13 @@ use activitystreams::{
   prelude::*,
 };
 use anyhow::Context;
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   private_message::{PrivateMessage, PrivateMessageForm},
   user::User_,
   Crud,
 };
-use lemmy_utils::{convert_datetime, location_info, LemmyError};
+use lemmy_utils::{location_info, utils::convert_datetime, LemmyError};
 use url::Url;
 
 #[async_trait::async_trait(?Send)]

src/apub/user.rs

@@ -1,5 +1,4 @@
 use crate::{
-  api::{check_slurs, check_slurs_opt},
   apub::{
     activities::generate_activity_id,
     activity_queue::send_activity,
@@ -12,7 +11,6 @@ use crate::{
     PersonExt,
     ToApub,
   },
-  blocking,
   DbPool,
   LemmyContext,
 };
@@ -29,11 +27,16 @@ use activitystreams::{
 use activitystreams_ext::Ext1;
 use actix_web::{body::Body, web, HttpResponse};
 use anyhow::Context;
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   naive_now,
   user::{UserForm, User_},
 };
-use lemmy_utils::{convert_datetime, location_info, LemmyError};
+use lemmy_utils::{
+  location_info,
+  utils::{check_slurs, check_slurs_opt, convert_datetime},
+  LemmyError,
+};
 use serde::Deserialize;
 use url::Url;
 

src/code_migrations.rs

@@ -13,11 +13,8 @@ use lemmy_db::{
   Crud,
 };
 use lemmy_utils::{
-  generate_actor_keypair,
+  apub::{generate_actor_keypair, get_apub_protocol_string, make_apub_endpoint, EndpointType},
-  get_apub_protocol_string,
-  make_apub_endpoint,
   settings::Settings,
-  EndpointType,
   LemmyError,
 };
 use log::info;

src/lib.rs

@@ -36,15 +36,14 @@ use crate::{
 use actix::Addr;
 use anyhow::anyhow;
 use background_jobs::QueueHandle;
-use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError};
+use lemmy_db::DbPool;
+use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
 use log::error;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use reqwest::Client;
 use serde::Deserialize;
 use std::process::Command;
 
-pub type DbPool = diesel::r2d2::Pool<diesel::r2d2::ConnectionManager<diesel::PgConnection>>;
-
 pub struct LemmyContext {
   pub pool: DbPool,
   pub chat_server: Addr<ChatServer>,
@@ -224,22 +223,6 @@ pub async fn is_image_content_type(client: &Client, test: &str) -> Result<(), Le
   }
 }
 
-pub async fn blocking<F, T>(pool: &DbPool, f: F) -> Result<T, LemmyError>
-where
-  F: FnOnce(&diesel::PgConnection) -> T + Send + 'static,
-  T: Send + 'static,
-{
-  let pool = pool.clone();
-  let res = actix_web::web::block(move || {
-    let conn = pool.get()?;
-    let res = (f)(&conn);
-    Ok(res) as Result<_, LemmyError>
-  })
-  .await?;
-
-  Ok(res)
-}
-
 pub fn captcha_espeak_wav_base64(captcha: &str) -> Result<String, LemmyError> {
   let mut built_text = String::new();
 

src/main.rs

@@ -16,11 +16,11 @@ use diesel::{
   PgConnection,
 };
 use lazy_static::lazy_static;
+use lemmy_api_structs::blocking;
 use lemmy_db::get_database_url_from_env;
 use lemmy_rate_limit::{rate_limiter::RateLimiter, RateLimit};
 use lemmy_server::{
   apub::activity_queue::create_activity_queue,
-  blocking,
   code_migrations::run_advanced_migrations,
   routes::*,
   websocket::chat_server::ChatServer,

src/routes/feeds.rs

@@ -1,8 +1,9 @@
-use crate::{api::claims::Claims, blocking, LemmyContext};
+use crate::{api::claims::Claims, LemmyContext};
 use actix_web::{error::ErrorBadRequest, *};
 use anyhow::anyhow;
 use chrono::{DateTime, NaiveDateTime, Utc};
 use diesel::PgConnection;
+use lemmy_api_structs::blocking;
 use lemmy_db::{
   comment_view::{ReplyQueryBuilder, ReplyView},
   community::Community,
@@ -13,7 +14,7 @@ use lemmy_db::{
   ListingType,
   SortType,
 };
-use lemmy_utils::{markdown_to_html, settings::Settings, LemmyError};
+use lemmy_utils::{settings::Settings, utils::markdown_to_html, LemmyError};
 use rss::{CategoryBuilder, ChannelBuilder, GuidBuilder, Item, ItemBuilder};
 use serde::Deserialize;
 use std::str::FromStr;

src/routes/nodeinfo.rs

@@ -1,8 +1,9 @@
-use crate::{blocking, version, LemmyContext};
+use crate::{version, LemmyContext};
 use actix_web::{body::Body, error::ErrorBadRequest, *};
 use anyhow::anyhow;
+use lemmy_api_structs::blocking;
 use lemmy_db::site_view::SiteView;
-use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError};
+use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
 use serde::{Deserialize, Serialize};
 use url::Url;
 

src/routes/webfinger.rs

@@ -1,6 +1,7 @@
-use crate::{blocking, LemmyContext};
+use crate::LemmyContext;
 use actix_web::{error::ErrorBadRequest, web::Query, *};
 use anyhow::anyhow;
+use lemmy_api_structs::blocking;
 use lemmy_db::{community::Community, user::User_};
 use lemmy_utils::{
   settings::Settings,

src/routes/websocket.rs

@@ -8,7 +8,7 @@ use crate::{
 use actix::prelude::*;
 use actix_web::*;
 use actix_web_actors::ws;
-use lemmy_utils::get_ip;
+use lemmy_utils::utils::get_ip;
 use log::{debug, error, info};
 use std::time::{Duration, Instant};
 

test.sh

@@ -2,4 +2,4 @@
 export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
 diesel migration run
 export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
-RUST_TEST_THREADS=1 cargo test --workspace
+RUST_TEST_THREADS=1 cargo test --workspace --no-fail-fast