Merge pull request #422 from calzoneman/cytubefilters

Switch to PCRE-based C++ chat filters

lib/channel/chat.js

@@ -315,7 +315,7 @@ ChatModule.prototype.filterMessage = function (msg) {
         /* substring is a URL */
         if (convertLinks && parts[j].match(link)) {
             var original = parts[j];
-            parts[j] = filters.exec(parts[j], { filterlinks: true });
+            parts[j] = filters.filter(parts[j], true);
 
             /* no filters changed the URL, apply link filter */
             if (parts[j] === original) {
@@ -325,7 +325,7 @@ ChatModule.prototype.filterMessage = function (msg) {
 
         } else {
             /* substring is not a URL */
-            parts[j] = filters.exec(parts[j], { filterlinks: false });
+            parts[j] = filters.filter(parts[j], false);
         }
     }
 

lib/channel/filters.js

@@ -1,169 +1,89 @@
+var FilterList = require("cytubefilters");
 var ChannelModule = require("./module");
 var XSS = require("../xss");
+var Logger = require("../logger");
 
-function ChatFilter(name, regex, flags, replace, active, filterlinks) {
+/*
-    this.name = name;
+ * Converts JavaScript-style replacements ($1, $2, etc.) with
-    this.source = regex;
+ * PCRE-style (\1, \2, etc.)
-    this.flags = flags;
+ */
-    this.regex = new RegExp(this.source, flags);
+function fixReplace(replace) {
-    this.replace = replace;
+    return replace.replace(/\$(\d)/g, "\\$1");
-    this.active = active === false ? false : true;
-    this.filterlinks = filterlinks || false;
 }
 
-ChatFilter.prototype = {
-    pack: function () {
-        return {
-            name: this.name,
-            source: this.source,
-            flags: this.flags,
-            replace: this.replace,
-            active: this.active,
-            filterlinks: this.filterlinks
-        };
-    },
-
-    exec: function (str) {
-        return str.replace(this.regex, this.replace);
-    }
-};
-
-function FilterList(defaults) {
-    if (!defaults) {
-        defaults = [];
-    }
-
-    this.filters = defaults.map(function (f) {
-        return new ChatFilter(f.name, f.source, f.flags, f.replace, f.active, f.filterlinks);
-    });
-}
-
-FilterList.prototype = {
-    pack: function () {
-        return this.filters.map(function (f) { return f.pack(); });
-    },
-
-    importList: function (filters) {
-        this.filters = Array.prototype.slice.call(filters);
-    },
-
-    updateFilter: function (filter) {
-        if (!filter.name) {
-            filter.name = filter.source;
-        }
-
-        var found = false;
-        for (var i = 0; i < this.filters.length; i++) {
-            if (this.filters[i].name === filter.name) {
-                found = true;
-                this.filters[i] = filter;
-                break;
-            }
-        }
-
-        /* If no filter was updated, add a new one */
-        if (!found) {
-            this.filters.push(filter);
-        }
-    },
-
-    removeFilter: function (filter) {
-        var found = false;
-        for (var i = 0; i < this.filters.length; i++) {
-            if (this.filters[i].name === filter.name) {
-                this.filters.splice(i, 1);
-                break;
-            }
-        }
-    },
-
-    moveFilter: function (from, to) {
-        if (from < 0 || to < 0 ||
-            from >= this.filters.length || to >= this.filters.length) {
-            return false;
-        }
-
-        var f = this.filters[from];
-        /* Offset from/to indexes to account for the fact that removing
-           an element changes the position of one of them.
-
-           I could have just done a swap, but it's already implemented this way
-           and it works. */
-        to = to > from ? to + 1 : to;
-        from = to > from ? from : from + 1;
-
-        this.filters.splice(to, 0, f);
-        this.filters.splice(from, 1);
-        return true;
-    },
-
-    exec: function (str, opts) {
-        if (!opts) {
-            opts = {};
-        }
-
-        this.filters.forEach(function (f) {
-            if (opts.filterlinks && !f.filterlinks) {
-                return;
-            }
-
-            if (f.active) {
-                str = f.exec(str);
-            }
-        });
-
-        return str;
-    }
-};
-
 function validateFilter(f) {
     if (typeof f.source !== "string" || typeof f.flags !== "string" ||
         typeof f.replace !== "string") {
-        return false;
+        return null;
     }
 
     if (typeof f.name !== "string") {
         f.name = f.source;
     }
 
-    f.replace = f.replace.substring(0, 1000);
+    f.replace = fixReplace(f.replace.substring(0, 1000));
     f.replace = XSS.sanitizeHTML(f.replace);
     f.flags = f.flags.substring(0, 4);
 
     try {
-        new RegExp(f.source, f.flags);
+        FilterList.checkValidRegex(f.source);
     } catch (e) {
-        return false;
+        return null;
     }
 
-    var filter = new ChatFilter(f.name, f.source, f.flags, f.replace,
+    var filter = {
-                                Boolean(f.active), Boolean(f.filterlinks));
+        name: f.name,
+        source: f.source,
+        replace: fixReplace(f.replace),
+        flags: f.flags,
+        active: !!f.active,
+        filterlinks: !!f.filterlinks
+    };
+
     return filter;
 }
 
+function makeDefaultFilter(name, source, flags, replace) {
+    return {
+        name: name,
+        source: source,
+        flags: flags,
+        replace: replace,
+        active: true,
+        filterlinks: false
+    };
+}
+
 const DEFAULT_FILTERS = [
-    new ChatFilter("monospace", "`(.+?)`", "g", "<code>$1</code>"),
+    makeDefaultFilter("monospace", "`(.+?)`", "g", "<code>\\1</code>"),
-    new ChatFilter("bold", "\\*(.+?)\\*", "g", "<strong>$1</strong>"),
+    makeDefaultFilter("bold", "\\*(.+?)\\*", "g", "<strong>\\1</strong>"),
-    new ChatFilter("italic", "_(.+?)_", "g", "<em>$1</em>"),
+    makeDefaultFilter("italic", "_(.+?)_", "g", "<em>\\1</em>"),
-    new ChatFilter("strike", "~~(.+?)~~", "g", "<s>$1</s>"),
+    makeDefaultFilter("strike", "~~(.+?)~~", "g", "<s>\\1</s>"),
-    new ChatFilter("inline spoiler", "\\[sp\\](.*?)\\[\\/sp\\]", "ig", "<span class=\"spoiler\">$1</span>")
+    makeDefaultFilter("inline spoiler", "\\[sp\\](.*?)\\[\\/sp\\]", "ig",
+        "<span class=\"spoiler\">\\1</span>")
 ];
 
 function ChatFilterModule(channel) {
     ChannelModule.apply(this, arguments);
-    this.filters = new FilterList(DEFAULT_FILTERS);
+    this.filters = new FilterList();
 }
 
 ChatFilterModule.prototype = Object.create(ChannelModule.prototype);
 
 ChatFilterModule.prototype.load = function (data) {
     if ("filters" in data) {
-        for (var i = 0; i < data.filters.length; i++) {
+        var filters = data.filters.map(validateFilter).filter(function (f) {
-            var f = validateFilter(data.filters[i]);
+            return f !== null;
-            if (f) {
+        });
-                this.filters.updateFilter(f);
+        try {
-            }
+            this.filters = new FilterList(filters);
+        } catch (e) {
+            Logger.errlog.log("Filter load failed: " + e + " (channel:" +
+                this.channel.name);
+            this.channel.logger.log("Failed to load filters: " + e);
         }
+    } else {
+        this.filters = new FilterList(DEFAULT_FILTERS);
     }
 };
 
@@ -173,11 +93,12 @@ ChatFilterModule.prototype.save = function (data) {
 
 ChatFilterModule.prototype.packInfo = function (data, isAdmin) {
     if (isAdmin) {
-        data.chatFilterCount = this.filters.filters.length;
+        data.chatFilterCount = this.filters.length;
     }
 };
 
 ChatFilterModule.prototype.onUserPostJoin = function (user) {
+    user.socket.on("addFilter", this.handleAddFilter.bind(this, user));
     user.socket.on("updateFilter", this.handleUpdateFilter.bind(this, user));
     user.socket.on("importFilters", this.handleImportFilters.bind(this, user));
     user.socket.on("moveFilter", this.handleMoveFilter.bind(this, user));
@@ -195,6 +116,54 @@ ChatFilterModule.prototype.sendChatFilters = function (users) {
     });
 };
 
+ChatFilterModule.prototype.handleAddFilter = function (user, data) {
+    if (typeof data !== "object") {
+        return;
+    }
+
+    if (!this.channel.modules.permissions.canEditFilters(user)) {
+        return;
+    }
+
+    try {
+        FilterList.checkValidRegex(data.source);
+    } catch (e) {
+        user.socket.emit("errorMsg", {
+            msg: "Invalid regex: " + e.message,
+            alert: true
+        });
+        return;
+    }
+
+    data = validateFilter(data);
+    if (!data) {
+        return;
+    }
+
+    try {
+        this.filters.addFilter(data);
+    } catch (e) {
+        user.socket.emit("errorMsg", {
+            msg: "Filter add failed: " + e.message,
+            alert: true
+        });
+        return;
+    }
+
+    user.socket.emit("addFilterSuccess");
+
+    var chan = this.channel;
+    chan.users.forEach(function (u) {
+        if (chan.modules.permissions.canEditFilters(u)) {
+            u.socket.emit("updateChatFilter", data);
+        }
+    });
+
+    chan.logger.log("[mod] " + user.getName() + " added filter: " + data.name + " -> " +
+                    "s/" + data.source + "/" + data.replace + "/" + data.flags +
+                    " active: " + data.active + ", filterlinks: " + data.filterlinks);
+};
+
 ChatFilterModule.prototype.handleUpdateFilter = function (user, data) {
     if (typeof data !== "object") {
         return;
@@ -204,13 +173,31 @@ ChatFilterModule.prototype.handleUpdateFilter = function (user, data) {
         return;
     }
 
-    var f = validateFilter(data);
+    try {
-    if (!f) {
+        FilterList.checkValidRegex(data.source);
+    } catch (e) {
+        user.socket.emit("errorMsg", {
+            msg: "Invalid regex: " + e.message,
+            alert: true
+        });
+        return;
+    }
+
+    data = validateFilter(data);
+    if (!data) {
+        return;
+    }
+
+    try {
+        this.filters.updateFilter(data);
+    } catch (e) {
+        user.socket.emit("errorMsg", {
+            msg: "Filter update failed: " + e.message,
+            alert: true
+        });
         return;
     }
-    data = f.pack();
 
-    this.filters.updateFilter(f);
     var chan = this.channel;
     chan.users.forEach(function (u) {
         if (chan.modules.permissions.canEditFilters(u)) {
@@ -218,9 +205,9 @@ ChatFilterModule.prototype.handleUpdateFilter = function (user, data) {
         }
     });
 
-    chan.logger.log("[mod] " + user.getName() + " updated filter: " + f.name + " -> " +
+    chan.logger.log("[mod] " + user.getName() + " updated filter: " + data.name + " -> " +
-                    "s/" + f.source + "/" + f.replace + "/" + f.flags + " active: " +
+                    "s/" + data.source + "/" + data.replace + "/" + data.flags +
-                    f.active + ", filterlinks: " + f.filterlinks);
+                    " active: " + data.active + ", filterlinks: " + data.filterlinks);
 };
 
 ChatFilterModule.prototype.handleImportFilters = function (user, data) {
@@ -234,9 +221,17 @@ ChatFilterModule.prototype.handleImportFilters = function (user, data) {
         return;
     }
 
-    this.filters.importList(data.map(validateFilter).filter(function (f) {
+    try {
-        return f !== false;
+        this.filters = new FilterList(data.map(validateFilter).filter(function (f) {
-    }));
+            return f !== null;
+        }));
+    } catch (e) {
+        user.socket.emit("errorMsg", {
+            msg: "Filter import failed: " + e.message,
+            alert: true
+        });
+        return;
+    }
 
     this.channel.logger.log("[mod] " + user.getName() + " imported the filter list");
     this.sendChatFilters(this.channel.users);
@@ -255,13 +250,22 @@ ChatFilterModule.prototype.handleRemoveFilter = function (user, data) {
         return;
     }
 
-    this.filters.removeFilter(data);
+    try {
+        this.filters.removeFilter(data);
+    } catch (e) {
+        user.socket.emit("errorMsg", {
+            msg: "Filter removal failed: " + e.message,
+            alert: true
+        });
+        return;
+    }
     var chan = this.channel;
     chan.users.forEach(function (u) {
         if (chan.modules.permissions.canEditFilters(u)) {
             u.socket.emit("deleteChatFilter", data);
         }
     });
+
     this.channel.logger.log("[mod] " + user.getName() + " removed filter: " + data.name);
 };
 
@@ -278,7 +282,15 @@ ChatFilterModule.prototype.handleMoveFilter = function (user, data) {
         return;
     }
 
-    this.filters.moveFilter(data.from, data.to);
+    try {
+        this.filters.moveFilter(data.from, data.to);
+    } catch (e) {
+        user.socket.emit("errorMsg", {
+            msg: "Filter move failed: " + e.message,
+            alert: true
+        });
+        return;
+    }
 };
 
 ChatFilterModule.prototype.handleRequestChatFilters = function (user) {

package.json

@@ -11,6 +11,7 @@
     "body-parser": "^1.6.5",
     "compression": "^1.2.0",
     "cookie-parser": "^1.3.2",
+    "cytubefilters": "git://github.com/calzoneman/cytubefilters#a5a99642",
     "express": "^4.8.5",
     "express-minify": "0.0.11",
     "jade": "^1.5.0",

www/js/ui.js

@@ -621,14 +621,7 @@ $("#cs-chatfilters-newsubmit").click(function () {
               "match.");
     }
 
-    try {
+    socket.emit("addFilter", {
-        new RegExp(regex, flags);
-    } catch (e) {
-        alert("Regex error: " + e);
-        return;
-    }
-
-    socket.emit("updateFilter", {
         name: name,
         source: regex,
         flags: flags,
@@ -636,10 +629,12 @@ $("#cs-chatfilters-newsubmit").click(function () {
         active: true
     });
 
-    $("#cs-chatfilters-newname").val("");
+    socket.once("addFilterSuccess", function () {
-    $("#cs-chatfilters-newregex").val("");
+        $("#cs-chatfilters-newname").val("");
-    $("#cs-chatfilters-newflags").val("");
+        $("#cs-chatfilters-newregex").val("");
-    $("#cs-chatfilters-newreplace").val("");
+        $("#cs-chatfilters-newflags").val("");
+        $("#cs-chatfilters-newreplace").val("");
+    });
 });
 
 $("#cs-emotes-newsubmit").click(function () {

www/js/util.js

@@ -2304,14 +2304,11 @@ function formatCSChatFilterList() {
                 f.flags = flags.val();
                 f.replace = replace.val();
                 f.filterlinks = filterlinks.prop("checked");
-                try {
-                    new RegExp(f.source, f.flags);
-                } catch (e) {
-                    alert("Invalid regex: " + e);
-                }
 
                 socket.emit("updateFilter", f);
-                reset();
+                socket.once("updateFilterSuccess", function () {
+                    reset();
+                });
             });
 
             control.data("editor", tr2);