From c7bec6251e904ff19e6bfd7dc08be28e0327162f Mon Sep 17 00:00:00 2001 From: Calvin Montgomery Date: Sun, 16 Jul 2017 22:35:33 -0700 Subject: [PATCH] Begin prometheus integration Add a dependency on `prom-client` and emit a basic latency metric for testing purposes. Add a new configuration file for enabling/disabling prometheus exporter and configuring the listen address. --- conf/example/prometheus.toml | 14 ++++++ package.json | 1 + src/config.js | 29 ++++++++++++ src/configuration/prometheusconfig.js | 23 +++++++++ src/get-info.js | 1 + src/prometheus-server.js | 47 +++++++++++++++++++ src/server.js | 6 +++ src/web/webserver.js | 27 ++++++++++- test/configuration/prometheusconfig.js | 10 ++++ test/prometheus-server.js | 65 ++++++++++++++++++++++++++ 10 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 conf/example/prometheus.toml create mode 100644 src/configuration/prometheusconfig.js create mode 100644 src/prometheus-server.js create mode 100644 test/configuration/prometheusconfig.js create mode 100644 test/prometheus-server.js diff --git a/conf/example/prometheus.toml b/conf/example/prometheus.toml new file mode 100644 index 00000000..a0129984 --- /dev/null +++ b/conf/example/prometheus.toml @@ -0,0 +1,14 @@ +# Configuration for binding an HTTP server to export prometheus metrics. +# See https://prometheus.io/ and https://github.com/siimon/prom-client +# for more details. +[prometheus] +enabled = true +# Host, port to bind. This is separate from the main CyTube HTTP server +# because it may be desirable to bind a different IP/port for monitoring +# purposes. Default: localhost port 19820 (arbitrary port chosen not to +# conflict with existing prometheus exporters). +host = '127.0.0.1' +port = 19820 +# Request path to serve metrics. All other paths are rejected with +# 400 Bad Request. +path = '/metrics' diff --git a/package.json b/package.json index 3eb23ab7..87e1cc7f 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "mysql": "^2.9.0", "nodemailer": "^1.4.0", "oauth": "^0.9.12", + "prom-client": "^10.0.2", "proxy-addr": "^1.1.4", "pug": "^2.0.0-beta3", "q": "^1.4.1", diff --git a/src/config.js b/src/config.js index 86831025..2cff9b76 100644 --- a/src/config.js +++ b/src/config.js @@ -6,6 +6,7 @@ var YAML = require("yamljs"); import { loadFromToml } from 'cytube-common/lib/configuration/configloader'; import { CamoConfig } from './configuration/camoconfig'; +import { PrometheusConfig } from './configuration/prometheusconfig'; const LOGGER = require('@calzoneman/jsli')('config'); @@ -149,6 +150,7 @@ function merge(obj, def, path) { var cfg = defaults; let camoConfig = new CamoConfig(); +let prometheusConfig = new PrometheusConfig(); /** * Initializes the configuration from the given YAML file @@ -191,6 +193,7 @@ exports.load = function (file) { LOGGER.info("Loaded configuration from " + file); loadCamoConfig(); + loadPrometheusConfig(); }; function loadCamoConfig() { @@ -214,6 +217,28 @@ function loadCamoConfig() { } } +function loadPrometheusConfig() { + try { + prometheusConfig = loadFromToml(PrometheusConfig, + path.resolve(__dirname, '..', 'conf', 'prometheus.toml')); + const enabled = prometheusConfig.isEnabled() ? 'ENABLED' : 'DISABLED'; + LOGGER.info('Loaded prometheus configuration from conf/prometheus.toml. ' + + `Prometheus listener is ${enabled}`); + } catch (error) { + if (error.code === 'ENOENT') { + LOGGER.info('No prometheus configuration found, defaulting to disabled'); + prometheusConfig = new PrometheusConfig(); + return; + } + + if (typeof error.line !== 'undefined') { + LOGGER.error(`Error in conf/prometheus.toml: ${error} (line ${error.line})`); + } else { + LOGGER.error(`Error loading conf/prometheus.toml: ${error.stack}`); + } + } +} + // I'm sorry function preprocessConfig(cfg) { /* Detect 3.0.0-style config and warng the user about it */ @@ -483,3 +508,7 @@ exports.set = function (key, value) { exports.getCamoConfig = function getCamoConfig() { return camoConfig; }; + +exports.getPrometheusConfig = function getPrometheusConfig() { + return prometheusConfig; +}; diff --git a/src/configuration/prometheusconfig.js b/src/configuration/prometheusconfig.js new file mode 100644 index 00000000..af45aef1 --- /dev/null +++ b/src/configuration/prometheusconfig.js @@ -0,0 +1,23 @@ +class PrometheusConfig { + constructor(config = { prometheus: { enabled: false } }) { + this.config = config.prometheus; + } + + isEnabled() { + return this.config.enabled; + } + + getPort() { + return this.config.port; + } + + getHost() { + return this.config.host; + } + + getPath() { + return this.config.path; + } +} + +export { PrometheusConfig }; diff --git a/src/get-info.js b/src/get-info.js index 67acb70d..86a23760 100644 --- a/src/get-info.js +++ b/src/get-info.js @@ -588,6 +588,7 @@ module.exports = { Getters: Getters, getMedia: function (id, type, callback) { if(type in this.Getters) { + LOGGER.info("Looking up %s:%s", type, id); this.Getters[type](id, callback); } else { callback("Unknown media type '" + type + "'", null); diff --git a/src/prometheus-server.js b/src/prometheus-server.js new file mode 100644 index 00000000..f94fca57 --- /dev/null +++ b/src/prometheus-server.js @@ -0,0 +1,47 @@ +import http from 'http'; +import { register } from 'prom-client'; +import { parse as parseURL } from 'url'; + +const LOGGER = require('@calzoneman/jsli')('prometheus-server'); + +let server = null; + +export function init(prometheusConfig) { + if (server !== null) { + LOGGER.error('init() called but server is already initialized! %s', + new Error().stack); + return; + } + + server = http.createServer((req, res) => { + if (req.method !== 'GET' + || parseURL(req.url).pathname !== prometheusConfig.getPath()) { + res.writeHead(400, { 'Content-Type': 'text/plain' }); + res.end('Bad Request'); + return; + } + + res.writeHead(200, { + 'Content-Type': register.contentType + }); + res.end(register.metrics()); + }); + + server.on('error', error => { + LOGGER.error('Server error: %s', error.stack); + }); + + server.once('listening', () => { + LOGGER.info('Prometheus metrics reporter listening on %s:%s', + prometheusConfig.getHost(), + prometheusConfig.getPort()); + }); + + server.listen(prometheusConfig.getPort(), prometheusConfig.getHost()); + return { once: server.once.bind(server) }; +} + +export function shutdown() { + server.close(); + server = null; +} diff --git a/src/server.js b/src/server.js index 8b4a2331..17bbbd22 100644 --- a/src/server.js +++ b/src/server.js @@ -147,6 +147,12 @@ var Server = function () { // background tasks init ---------------------------------------------- require("./bgtask")(self); + // prometheus server + const prometheusConfig = Config.getPrometheusConfig(); + if (prometheusConfig.isEnabled()) { + require("./prometheus-server").init(prometheusConfig); + } + // setuid require("./setuid"); diff --git a/src/web/webserver.js b/src/web/webserver.js index fda1d156..0e46bf92 100644 --- a/src/web/webserver.js +++ b/src/web/webserver.js @@ -10,7 +10,8 @@ import morgan from 'morgan'; import csrf from './csrf'; import * as HTTPStatus from './httpstatus'; import { CSRFError, HTTPError } from '../errors'; -import counters from "../counters"; +import counters from '../counters'; +import { Summary } from 'prom-client'; const LOGGER = require('@calzoneman/jsli')('webserver'); @@ -27,6 +28,29 @@ function initializeLog(app) { })); } +function initPrometheus(app) { + const latency = new Summary({ + name: 'cytube_http_req_latency', + help: 'HTTP Request latency from execution of the first middleware ' + + 'until the "finish" event on the response object.', + labelNames: ['method', 'statusCode'] + }); + + app.use((req, res, next) => { + const startTime = process.hrtime(); + res.on('finish', () => { + try { + const diff = process.hrtime(startTime); + const diffMs = diff[0]*1e3 + diff[1]*1e-6; + latency.labels(req.method, res.statusCode).observe(diffMs); + } catch (error) { + LOGGER.error('Failed to record HTTP Prometheus metrics: %s', error.stack); + } + }); + next(); + }); +} + /** * Redirects a request to HTTPS if the server supports it */ @@ -133,6 +157,7 @@ module.exports = { init: function (app, webConfig, ioConfig, clusterClient, channelIndex, session) { const chanPath = Config.get('channel-path'); + initPrometheus(app); app.use((req, res, next) => { counters.add("http:request", 1); next(); diff --git a/test/configuration/prometheusconfig.js b/test/configuration/prometheusconfig.js new file mode 100644 index 00000000..cd9a9139 --- /dev/null +++ b/test/configuration/prometheusconfig.js @@ -0,0 +1,10 @@ +const assert = require('assert'); +const PrometheusConfig = require('../../lib/configuration/prometheusconfig').PrometheusConfig; + +describe('PrometheusConfig', () => { + describe('#constructor', () => { + it('defaults to enabled=false', () => { + assert.strictEqual(new PrometheusConfig().isEnabled(), false); + }); + }); +}); diff --git a/test/prometheus-server.js b/test/prometheus-server.js new file mode 100644 index 00000000..f7212921 --- /dev/null +++ b/test/prometheus-server.js @@ -0,0 +1,65 @@ +const assert = require('assert'); +const http = require('http'); +const server = require('../lib/prometheus-server'); +const PrometheusConfig = require('../lib/configuration/prometheusconfig').PrometheusConfig; + +describe('prometheus-server', () => { + before(done => { + let inst = server.init(new PrometheusConfig({ + prometheus: { + enabled: true, + port: 19820, + host: '127.0.0.1', + path: '/metrics' + } + })); + inst.once('listening', () => done()); + }); + + function checkReq(options, done) { + const req = http.request({ + method: options.method, + host: '127.0.0.1', + port: 19820, + path: options.path + }, res => { + assert.strictEqual(res.statusCode, options.expectedStatusCode); + assert.strictEqual(res.headers['content-type'], options.expectedContentType); + res.on('data', () => {}); + res.on('end', () => done()); + }); + + req.end(); + } + + it('rejects a non-GET request', done => { + checkReq({ + method: 'POST', + path: '/metrics', + expectedStatusCode: 400, + expectedContentType: 'text/plain' + }, done); + }); + + it('rejects a request for the wrong path', done => { + checkReq({ + method: 'GET', + path: '/qwerty', + expectedStatusCode: 400, + expectedContentType: 'text/plain' + }, done); + }); + + it('accepts a request for the configured path', done => { + checkReq({ + method: 'GET', + path: '/metrics', + expectedStatusCode: 200, + expectedContentType: 'text/plain; version=0.0.4; charset=utf-8' + }, done); + }); + + after(() => { + server.shutdown(); + }); +});