Remove old flatfile chandump storage

NEWS.md

@@ -1,3 +1,33 @@
+2020-02-15
+==========
+
+Old versions of CyTube defaulted to storing channel state in flatfiles located
+in the `chandump` directory.  The default was changed a while ago, and the
+flatfile storage mechanism has now been removed.
+
+Admins who have not already migrated their installation to the "database"
+channel storage type can do so by following these instructions:
+
+  1. Run `git checkout e3a9915b454b32e49d3871c94c839899f809520a` to temporarily
+     switch to temporarily revert to the previous version of the code that
+     supports the "file" channel storage type
+  2. Run `npm run build-server` to build the old version
+  3. Run `node lib/channel-storage/migrator.js |& tee migration.log` to migrate
+     channel state from files to the database
+  4. Inspect the output of the migration tool for errors
+  5. Set `channel-storage`/`type` to `"database"` in `config.yaml` and start the
+     server.  Load a channel to verify the migration worked as expected
+  6. Upgrade back to the latest version with `git checkout 3.0` and `npm run
+     build-server`
+  7. Remove the `channel-storage` block from `config.yaml` and remove the
+     `chandump` directory since it is no longer needed (you may wish to archive
+     it somewhere in case you later discover the migration didn't work as
+     expected).
+
+If you encounter any errors during the process, please file an issue on GitHub
+and attach the output of the migration tool (which if you use the above commands
+will be written to `migration.log`).
+
 2019-12-01
 ==========
 

config.template.yaml

@@ -133,9 +133,6 @@ channel-path: 'r'
 channel-blacklist: []
 # Minutes between saving channel state to disk
 channel-save-interval: 5
-# Determines channel data storage mechanism.
-channel-storage:
-  type: 'database'
 
 # Configure periodic clearing of old alias data
 aliases:

package.json

@@ -2,7 +2,7 @@
   "author": "Calvin Montgomery",
   "name": "CyTube",
   "description": "Online media synchronizer and chat",
-  "version": "3.68.0",
+  "version": "3.69.0",
   "repository": {
     "url": "http://github.com/calzoneman/sync"
   },

src/channel-storage/channelstore.js

@@ -1,4 +1,3 @@
-import { FileStore } from './filestore';
 import { DatabaseStore } from './dbstore';
 import Config from '../config';
 import Promise from 'bluebird';
@@ -26,11 +25,12 @@ export function save(id, channelName, data) {
 }
 
 function loadChannelStore() {
-    switch (Config.get('channel-storage.type')) {
+    if (Config.get('channel-storage.type') === 'file') {
-        case 'database':
+        throw new Error(
-            return new DatabaseStore();
+            'channel-storage type "file" is no longer supported.  Please see ' +
-        case 'file':
+            'NEWS.md for instructions on upgrading.'
-        default:
+        );
-            return new FileStore();
     }
+
+    return new DatabaseStore();
 }

src/channel-storage/filestore.js

@@ -1,78 +0,0 @@
-import Promise from 'bluebird';
-import { stat } from 'fs';
-import * as fs from 'graceful-fs';
-import path from 'path';
-import { ChannelStateSizeError } from '../errors';
-
-const readFileAsync = Promise.promisify(fs.readFile);
-const writeFileAsync = Promise.promisify(fs.writeFile);
-const readdirAsync = Promise.promisify(fs.readdir);
-const statAsync = Promise.promisify(stat);
-const SIZE_LIMIT = 1048576;
-const CHANDUMP_DIR = path.resolve(__dirname, '..', '..', 'chandump');
-
-export class FileStore {
-    filenameForChannel(channelName) {
-        return path.join(CHANDUMP_DIR, channelName);
-    }
-
-    load(id, channelName) {
-        const filename = this.filenameForChannel(channelName);
-        return statAsync(filename).then(stats => {
-            if (stats.size > SIZE_LIMIT) {
-                return Promise.reject(
-                    new ChannelStateSizeError(
-                        'Channel state file is too large',
-                        {
-                            limit: SIZE_LIMIT,
-                            actual: stats.size
-                        }
-                    )
-                );
-            } else {
-                return readFileAsync(filename);
-            }
-        }).then(fileContents => {
-            try {
-                return JSON.parse(fileContents);
-            } catch (e) {
-                return Promise.reject(new Error('Channel state file is not valid JSON: ' + e));
-            }
-        });
-    }
-
-    async save(id, channelName, data) {
-        let original;
-        try {
-            original = await this.load(id, channelName);
-        } catch (error) {
-            if (error.code !== 'ENOENT') {
-                throw error;
-            } else {
-                original = {};
-            }
-        }
-
-        Object.keys(data).forEach(key => {
-            original[key] = data[key];
-        });
-
-        const filename = this.filenameForChannel(channelName);
-        const fileContents = new Buffer(JSON.stringify(original), 'utf8');
-        if (fileContents.length > SIZE_LIMIT) {
-            throw new ChannelStateSizeError(
-                'Channel state size is too large',
-                {
-                    limit: SIZE_LIMIT,
-                    actual: fileContents.length
-                }
-            );
-        }
-
-        return await writeFileAsync(filename, fileContents);
-    }
-
-    listChannels() {
-        return readdirAsync(CHANDUMP_DIR);
-    }
-}

src/channel-storage/migrator.js

@@ -1,187 +0,0 @@
-import Config from '../config';
-import Promise from 'bluebird';
-import db from '../database';
-import { FileStore } from './filestore';
-import { DatabaseStore } from './dbstore';
-import { sanitizeHTML } from '../xss';
-import { ChannelNotFoundError } from '../errors';
-
-const lookupAsync = Promise.promisify(require('../database/channels').lookup);
-
-/* eslint no-console: off */
-
-const EXPECTED_KEYS = [
-    'chatbuffer',
-    'chatmuted',
-    'css',
-    'emotes',
-    'filters',
-    'js',
-    'motd',
-    'openPlaylist',
-    'opts',
-    'permissions',
-    'playlist',
-    'poll'
-];
-
-function fixOldChandump(data) {
-    const converted = {};
-    EXPECTED_KEYS.forEach(key => {
-        converted[key] = data[key];
-    });
-
-    if (data.queue) {
-        converted.playlist = {
-            pl: data.queue.map(item => {
-                return {
-                    media: {
-                        id: item.id,
-                        title: item.title,
-                        seconds: item.seconds,
-                        duration: item.duration,
-                        type: item.type,
-                        meta: {}
-                    },
-                    queueby: item.queueby,
-                    temp: item.temp
-                };
-            }),
-            pos: data.position,
-            time: data.currentTime
-        };
-    }
-
-    if (data.hasOwnProperty('openqueue')) {
-        converted.openPlaylist = data.openqueue;
-    }
-
-    if (data.hasOwnProperty('playlistLock')) {
-        converted.openPlaylist = !data.playlistLock;
-    }
-
-    if (data.chatbuffer) {
-        converted.chatbuffer = data.chatbuffer.map(entry => {
-            return {
-                username: entry.username,
-                msg: entry.msg,
-                meta: entry.meta || {
-                    addClass: entry.msgclass ? entry.msgclass : undefined
-                },
-                time: entry.time
-            };
-        });
-    }
-
-    if (data.motd && data.motd.motd) {
-        converted.motd = sanitizeHTML(data.motd.motd).replace(/\n/g, '<br>\n');
-    }
-
-    if (data.opts && data.opts.customcss) {
-        converted.opts.externalcss = data.opts.customcss;
-    }
-
-    if (data.opts && data.opts.customjs) {
-        converted.opts.externaljs = data.opts.customjs;
-    }
-
-    if (data.filters && data.filters.length > 0 && Array.isArray(data.filters[0])) {
-        converted.filters = data.filters.map(filter => {
-            let [source, replace, active] = filter;
-            return {
-                source: source,
-                replace: replace,
-                flags: 'g',
-                active: active,
-                filterlinks: false
-            };
-        });
-    }
-
-    return converted;
-}
-
-function migrate(src, dest, opts) {
-    const chanPath = Config.get('channel-path');
-
-    return src.listChannels().then(names => {
-        return Promise.reduce(names, (_, name) => {
-            // A long time ago there was a bug where CyTube would save a different
-            // chandump depending on the capitalization of the channel name in the URL.
-            // This was fixed, but there are still some really old chandumps with
-            // uppercase letters in the name.
-            //
-            // If another chandump exists which is all lowercase, then that one is
-            // canonical.  Otherwise, it's safe to load the existing capitalization,
-            // convert it, and save.
-            if (name !== name.toLowerCase()) {
-                if (names.indexOf(name.toLowerCase()) >= 0) {
-                    return Promise.resolve();
-                }
-            }
-
-            let id;
-            return lookupAsync(name).then(chan => {
-                id = chan.id;
-                return src.load(id, name);
-            }).then(data => {
-                data = fixOldChandump(data);
-                Object.keys(data).forEach(key => {
-                    if (opts.keyWhitelist.length > 0 &&
-                            opts.keyWhitelist.indexOf(key) < 0) {
-                        delete data[key];
-                    } else if (opts.keyBlacklist.length > 0 &&
-                            opts.keyBlacklist.indexOf(key) >= 0) {
-                        delete data[key];
-                    }
-                });
-                return dest.save(id, name, data);
-            }).then(() => {
-                console.log(`Migrated /${chanPath}/${name}`);
-            }).catch(ChannelNotFoundError, _err => {
-                console.log(`Skipping /${chanPath}/${name} (not present in the database)`);
-            }).catch(err => {
-                console.error(`Failed to migrate /${chanPath}/${name}: ${err.stack}`);
-            });
-        }, 0);
-    });
-}
-
-function loadOpts(argv) {
-    const opts = {
-        keyWhitelist: [],
-        keyBlacklist: []
-    };
-
-    for (let i = 0; i < argv.length; i++) {
-        if (argv[i] === '-w') {
-            opts.keyWhitelist = (argv[i+1] || '').split(',');
-            i++;
-        } else if (argv[i] === '-b') {
-            opts.keyBlacklist = (argv[i+1] || '').split(',');
-            i++;
-        }
-    }
-
-    return opts;
-}
-
-function main() {
-    Config.load('config.yaml');
-    db.init();
-    const src = new FileStore();
-    const dest = new DatabaseStore();
-    const opts = loadOpts(process.argv.slice(2));
-
-    Promise.delay(1000).then(() => {
-        return migrate(src, dest, opts);
-    }).then(() => {
-        console.log('Migration complete');
-        process.exit(0);
-    }).catch(err => {
-        console.error(`Migration failed: ${err.stack}`);
-        process.exit(1);
-    });
-}
-
-main();

src/channel/channel.js

@@ -174,23 +174,6 @@ Channel.prototype.initModules = function () {
     self.logger.log("[init] Loaded modules: " + inited.join(", "));
 };
 
-Channel.prototype.getDiskSize = function (cb) {
-    if (this._getDiskSizeTimeout > Date.now()) {
-        return cb(null, this._cachedDiskSize);
-    }
-
-    var self = this;
-    var file = path.join(__dirname, "..", "..", "chandump", self.uniqueName);
-    fs.stat(file, function (err, stats) {
-        if (err) {
-            return cb(err);
-        }
-
-        self._cachedDiskSize = stats.size;
-        cb(null, self._cachedDiskSize);
-    });
-};
-
 Channel.prototype.loadState = function () {
     /* Don't load from disk if not registered */
     if (!this.is(Flags.C_REGISTERED)) {

src/config.js

@@ -65,9 +65,6 @@ var defaults = {
     "channel-blacklist": [],
     "channel-path": "r",
     "channel-save-interval": 5,
-    "channel-storage": {
-        type: "file"
-    },
     "max-channels-per-user": 5,
     "max-accounts-per-ip": 5,
     "guest-login-delay": 60,
@@ -436,6 +433,10 @@ function preprocessConfig(cfg) {
         'bucket-capacity': cfg.io.throttle['in-rate-limit']
     }, cfg.io.throttle);
 
+    if (!cfg['channel-storage']) {
+        cfg['channel-storage'] = { type: undefined };
+    }
+
     return cfg;
 }
 

src/database/channels.js

@@ -1,7 +1,5 @@
 var db = require("../database");
 var valid = require("../utilities").isValidChannelName;
-var fs = require("fs");
-var path = require("path");
 var Flags = require("../flags");
 var util = require("../utilities");
 import { createMySQLDuplicateKeyUpdate } from '../util/on-duplicate-key-update';
@@ -199,14 +197,6 @@ module.exports = {
                 }
             });
 
-            fs.unlink(path.join(__dirname, "..", "..", "chandump", name),
-                      function (err) {
-                if (err && err.code !== "ENOENT") {
-                    LOGGER.error("Deleting chandump failed:");
-                    LOGGER.error(err);
-                }
-            });
-
             callback(err, !err);
         });
     },

src/server.js

@@ -15,11 +15,6 @@ module.exports = {
             exists || fs.mkdirSync(chanlogpath);
         });
 
-        var chandumppath = path.join(__dirname, "../chandump");
-        fs.exists(chandumppath, function (exists) {
-            exists || fs.mkdirSync(chandumppath);
-        });
-
         var gdvttpath = path.join(__dirname, "../google-drive-subtitles");
         fs.exists(gdvttpath, function (exists) {
             exists || fs.mkdirSync(gdvttpath);

src/setuid.js

@@ -7,7 +7,6 @@ const LOGGER = require('@calzoneman/jsli')('setuid');
 
 var needPermissionsFixed = [
     path.join(__dirname, "..", "chanlogs"),
-    path.join(__dirname, "..", "chandump"),
     path.join(__dirname, "..", "google-drive-subtitles")
 ];