From e748d79349a955ec62b13aa6f6d797aedb97b336 Mon Sep 17 00:00:00 2001 From: calzoneman Date: Mon, 19 Aug 2013 23:53:33 -0500 Subject: [PATCH] Experiment with realtime stats of connection load --- acp.js | 12 +++++++ api.js | 16 +++++++++ server.js | 81 ++++++++++++++++++++++---------------------- stats.js | 37 ++++++++++++++++++++ www/acp.html | 16 +++++++++ www/assets/js/acp.js | 51 ++++++++++++++++++++++++++++ 6 files changed, 172 insertions(+), 41 deletions(-) diff --git a/acp.js b/acp.js index 0a332608..e817a914 100644 --- a/acp.js +++ b/acp.js @@ -169,6 +169,18 @@ module.exports = function (Server) { user.socket.emit("acp-view-stats", res); }); }); + + user.socket.on("acp-view-connstats", function () { + var http = Server.stats.readAverages("http"); + var sio = Server.stats.readAverages("socketio"); + var api = Server.stats.readAverages("api"); + + user.socket.emit("acp-view-connstats", { + http: http, + sio: sio, + api: api + }); + }); } } } diff --git a/api.js b/api.js index 2c92e8f0..0a9df9bc 100644 --- a/api.js +++ b/api.js @@ -55,6 +55,7 @@ module.exports = function (Server) { /* */ app.get("/api/coffee", function (req, res) { + Server.stats.record("api", "/api/coffee"); res.send(418); // 418 I'm a teapot }); @@ -62,6 +63,7 @@ module.exports = function (Server) { /* data about a specific channel */ app.get("/api/channels/:channel", function (req, res) { + Server.stats.record("api", "/api/channels/:channel"); var name = req.params.channel; if(!name.match(/^[\w-_]+$/)) { res.send(404); @@ -82,6 +84,7 @@ module.exports = function (Server) { /* data about all channels (filter= public or all) */ app.get("/api/allchannels/:filter", function (req, res) { + Server.stats.record("api", "/api/allchannels/:filter"); var filter = req.params.filter; if(filter !== "public" && filter !== "all") { res.send(400); @@ -140,6 +143,7 @@ module.exports = function (Server) { /* login */ app.post("/api/login", function (req, res) { + Server.stats.record("api", "/api/login"); res.type("application/jsonp"); res.setHeader("Access-Control-Allow-Origin", "*"); var name = req.body.name; @@ -181,6 +185,7 @@ module.exports = function (Server) { /* register an account */ app.post("/api/register", function (req, res) { + Server.stats.record("api", "/api/register"); res.type("application/jsonp"); res.setHeader("Access-Control-Allow-Origin", "*"); var name = req.body.name; @@ -252,6 +257,7 @@ module.exports = function (Server) { /* password change */ app.post("/api/account/passwordchange", function (req, res) { + Server.stats.record("api", "/api/account/passwordchange"); res.type("application/jsonp"); res.setHeader("Access-Control-Allow-Origin", "*"); @@ -295,6 +301,7 @@ module.exports = function (Server) { /* password reset */ app.post("/api/account/passwordreset", function (req, res) { + Server.stats.record("api", "/api/account/passwordreset"); res.type("application/jsonp"); res.setHeader("Access-Control-Allow-Origin", "*"); var name = req.body.name; @@ -364,6 +371,7 @@ module.exports = function (Server) { /* password recovery */ app.get("/api/account/passwordrecover", function (req, res) { + Server.stats.record("api", "/api/account/passwordrecover"); res.type("application/jsonp"); var hash = req.query.hash; var ip = getIP(req); @@ -388,6 +396,7 @@ module.exports = function (Server) { /* profile retrieval */ app.get("/api/users/:user/profile", function (req, res) { + Server.stats.record("api", "/api/users/:user/profile"); res.type("application/jsonp"); var name = req.params.user; @@ -410,6 +419,7 @@ module.exports = function (Server) { /* profile change */ app.post("/api/account/profile", function (req, res) { + Server.stats.record("api", "/api/account/profile"); res.type("application/jsonp"); res.setHeader("Access-Control-Allow-Origin", "*"); var name = req.body.name; @@ -457,6 +467,7 @@ module.exports = function (Server) { /* set email */ app.post("/api/account/email", function (req, res) { + Server.stats.record("api", "/api/account/email"); res.type("application/jsonp"); res.setHeader("Access-Control-Allow-Origin", "*"); var name = req.body.name; @@ -508,6 +519,7 @@ module.exports = function (Server) { /* my channels */ app.get("/api/account/mychannels", function (req, res) { + Server.stats.record("/api/account/mychannels"); res.type("application/jsonp"); var name = req.query.name; var session = req.query.session; @@ -545,6 +557,7 @@ module.exports = function (Server) { /* action log */ app.get("/api/logging/actionlog", function (req, res) { + Server.stats.record("api", "/api/logging/actionlog"); res.type("application/jsonp"); var name = req.query.name; var session = req.query.session; @@ -594,6 +607,7 @@ module.exports = function (Server) { } app.get("/api/logging/syslog", function (req, res) { + Server.stats.record("api", "/api/logging/syslog"); res.type("text/plain"); res.setHeader("Access-Control-Allow-Origin", "*"); @@ -621,6 +635,7 @@ module.exports = function (Server) { }); app.get("/api/logging/errorlog", function (req, res) { + Server.stats.record("api", "/api/logging/errorlog"); res.type("text/plain"); res.setHeader("Access-Control-Allow-Origin", "*"); @@ -648,6 +663,7 @@ module.exports = function (Server) { }); app.get("/api/logging/channels/:channel", function (req, res) { + Server.stats.record("api", "/api/logging/channels/:channel"); res.type("text/plain"); res.setHeader("Access-Control-Allow-Origin", "*"); diff --git a/server.js b/server.js index 23094c96..cbc4c091 100644 --- a/server.js +++ b/server.js @@ -90,49 +90,46 @@ var Server = { this.httpaccess.log([ipstr, req.method, url, status, req.headers["user-agent"]].join(" ")); }, init: function () { + var self = this; // init database var Database = require("./database"); - this.db = new Database(this.cfg); + this.db = new Database(self.cfg); this.db.init(); - this.actionlog = require("./actionlog")(this); + this.actionlog = require("./actionlog")(self); this.httpaccess = new Logger.Logger("httpaccess.log"); this.app = express(); this.app.use(express.bodyParser()); // channel path - this.app.get("/r/:channel(*)", function (req, res, next) { + self.app.get("/r/:channel(*)", function (req, res, next) { var c = req.params.channel; if(!c.match(/^[\w-_]+$/)) { res.redirect("/" + c); } else { - this.logHTTP(req); + self.stats.record("http", "/r/" + c); + self.logHTTP(req); res.sendfile(__dirname + "/www/channel.html"); } - }.bind(this)); + }); // api path - this.api = require("./api")(this); - /* - this.app.get("/api/:apireq(*)", function (req, res, next) { - this.logHTTP(req); - this.api.handle(req.url.substring(5), req, res); - }.bind(this)); - */ + self.api = require("./api")(self); - this.app.get("/", function (req, res, next) { - this.logHTTP(req); + self.app.get("/", function (req, res, next) { + self.logHTTP(req); + self.stats.record("http", "/"); res.sendfile(__dirname + "/www/index.html"); - }.bind(this)); + }); // default path - this.app.get("/:thing(*)", function (req, res, next) { + self.app.get("/:thing(*)", function (req, res, next) { var opts = { root: __dirname + "/www", - maxAge: this.cfg["asset-cache-ttl"] + maxAge: self.cfg["asset-cache-ttl"] } res.sendfile(req.params.thing, opts, function (err) { if(err) { - this.logHTTP(req, err.status); + self.logHTTP(req, err.status); // Damn path traversal attacks if(req.params.thing.indexOf("%2e") != -1) { res.send("Don't try that again, I'll ban you"); @@ -149,34 +146,36 @@ var Server = { } } else { - this.logHTTP(req); + self.stats.record("http", req.params.thing); + self.logHTTP(req); } - }.bind(this)); - }.bind(this)); + }); + }); // fallback - this.app.use(function (err, req, res, next) { - this.logHTTP(req, err.status); + self.app.use(function (err, req, res, next) { + self.logHTTP(req, err.status); if(err.status == 404) { res.send(404); } else { next(err); } - }.bind(this)); + }); // bind servers - this.httpserv = this.app.listen(Server.cfg["web-port"], + self.httpserv = self.app.listen(Server.cfg["web-port"], Server.cfg["express-host"]); - this.ioserv = express().listen(Server.cfg["io-port"], + self.ioserv = express().listen(Server.cfg["io-port"], Server.cfg["express-host"]); // init socket.io - this.io = require("socket.io").listen(this.ioserv); - this.io.set("log level", 1); - this.io.sockets.on("connection", function (socket) { + self.io = require("socket.io").listen(self.ioserv); + self.io.set("log level", 1); + self.io.sockets.on("connection", function (socket) { + self.stats.record("socketio", "socket"); var ip = getSocketIP(socket); socket._ip = ip; - this.db.isGlobalIPBanned(ip, function (err, bant) { + self.db.isGlobalIPBanned(ip, function (err, bant) { if(bant) { Logger.syslog.log("Disconnecting " + ip + " - gbanned"); socket.emit("kick", { @@ -187,14 +186,14 @@ var Server = { }); socket.on("disconnect", function () { - this.ips[ip]--; - }.bind(this)); + self.ips[ip]--; + }.bind(self)); - if(!(ip in this.ips)) - this.ips[ip] = 0; - this.ips[ip]++; + if(!(ip in self.ips)) + self.ips[ip] = 0; + self.ips[ip]++; - if(this.ips[ip] > Server.cfg["ip-connection-limit"]) { + if(self.ips[ip] > Server.cfg["ip-connection-limit"]) { socket.emit("kick", { reason: "Too many connections from your IP address" }); @@ -204,18 +203,18 @@ var Server = { // finally a valid user Logger.syslog.log("Accepted socket from /" + socket._ip); - new User(socket, this); - }.bind(this)); + new User(socket, self); + }.bind(self)); // init ACP - this.acp = require("./acp")(this); + self.acp = require("./acp")(self); // init stats - this.stats = require("./stats")(this); + self.stats = require("./stats")(self); // init media retriever - this.infogetter = require("./get-info")(this); + self.infogetter = require("./get-info")(self); }, shutdown: function () { Logger.syslog.log("Unloading channels"); diff --git a/stats.js b/stats.js index c8a061af..be2230f5 100644 --- a/stats.js +++ b/stats.js @@ -16,6 +16,7 @@ const STAT_EXPIRE = 24 * STAT_INTERVAL; module.exports = function (Server) { var db = Server.db; + setInterval(function () { var chancount = Server.channels.length; var usercount = 0; @@ -29,4 +30,40 @@ module.exports = function (Server) { db.pruneStats(Date.now() - STAT_EXPIRE); }); }, STAT_INTERVAL); + + return { + stores: { + "http": {}, + "socketio": {}, + "api": {} + }, + record: function (type, key) { + var store; + if(!(type in this.stores)) + return; + + store = this.stores[type]; + + if(key in store) { + store[key].push(Date.now()); + if(store[key].length > 100) + store[key].shift(); + } else { + store[key] = [Date.now()]; + } + }, + readAverages: function (type) { + if(!(type in this.stores)) + return; + var avg = {}; + var store = this.stores[type]; + for(var k in store) { + var time = Date.now() - store[k][0]; + avg[k] = store[k].length / time; + avg[k] = parseInt(avg[k] * 1000); + } + return avg; + } + }; + } diff --git a/www/acp.html b/www/acp.html index d362197f..fef0c5b7 100644 --- a/www/acp.html +++ b/www/acp.html @@ -46,6 +46,7 @@
  • Loaded Channels
  • Action Log
  • Server Stats
    • +
  • Connection Stats
    • @@ -249,6 +250,21 @@

    Memory Usage (MB)

    +
    +

    Connection Stats

    + Note: Data points for which the frequency average is 0 are not displayed. +
    + + + + + + + + + +
    TypeParamFrequency
    +
    diff --git a/www/assets/js/acp.js b/www/assets/js/acp.js index f96eb1f7..52a1eb06 100644 --- a/www/assets/js/acp.js +++ b/www/assets/js/acp.js @@ -147,6 +147,13 @@ menuHandler("#show_stats", "#stats"); $("#show_stats").click(function () { socket.emit("acp-view-stats"); }); +menuHandler("#show_connstats", "#connstats"); +$("#show_connstats").click(function () { + socket.emit("acp-view-connstats"); +}); +$("#connstats_refresh").click(function () { + socket.emit("acp-view-connstats"); +}); function reverseLog() { $("#log").text($("#log").text().split("\n").reverse().join("\n")); @@ -548,6 +555,50 @@ function setupCallbacks() { $("