CyTube/import.js

/**
 * Utility for importing a CyTube 2.4.6 database to 3.0
 */

var mysql = require("mysql");
var AsyncQueue = require("./lib/asyncqueue");
var tables = require("./lib/database/tables");

var olddb = {
    host: "",
    user: "",
    password: "",
    database: ""
};

var newdb = {
    host: "",
    user: "",
    password: "",
    database: ""
};

var oldpool;
var newpool;

function query(pool, query, sub, callback) {
    // 2nd argument is optional
    if (typeof sub === "function") {
        callback = sub;
        sub = false;
    }

    if (typeof callback !== "function") {
        callback = function () { };
    }

    pool.getConnection(function (err, conn) {
        if (err) {
            console.log("[ERROR] DB connection failed: " + err);
            callback("Database failure", null);
        } else {
            function cback(err, res) {
                if (err) {
                    console.log("[ERROR] DB query failed: " + query);
                    if (sub) {
                        console.log("[ERROR] Substitutions: " + sub);
                    }
                    console.log("[ERROR] " + err);
                    callback("Database failure", null);
                } else {
                    callback(null, res);
                }
                conn.release();
            }

            if (sub) {
                conn.query(query, sub, cback);
            } else {
                conn.query(query, cback);
            }
        }
    });
};

var queryOld;
var queryNew;

function chain(/* arguments */) {
    var args = Array.prototype.slice.call(arguments);
    var cb = args.pop();
    var next = function () {
        if (args.length > 0) {
            args.shift()(next);
        } else {
            cb();
        }
    };

    next();
}

/**
 * Imports entries from the registrations table of 2.4.6 to the users table of 3.0
 */
function importUsers(cb) {
    console.log("[INFO] Importing users");
    var insert = "INSERT INTO `users` (`name`, `password`, `global_rank`, " +
                 "`email`, `profile`, `time`) VALUES (?, ?, ?, ?, ?, ?)";
    queryOld("SELECT * FROM `registrations`", function (err, rows) {
        if (err) {
            cb(err);
            return;
        }

        rows.sort(function (a, b) {
            return a.id - b.id;
        });

        var aq = new AsyncQueue();
        rows.forEach(function (r) {
            var data = [r.uname, r.pw, r.global_rank, r.email,
                        JSON.stringify({ image: r.profile_image, text: r.profile_text }),
                        Date.now()];
            aq.queue(function (lock) {
                queryNew(insert, data, function (err) {
                    if (!err) {
                        console.log("Imported user " + r.uname);
                    }
                    lock.release();
                });
            });
        });

        aq.queue(function (lock) {
            lock.release();
            cb();
        });
    });
}

/**
 * Imports channel registration entries from `channels` table
 */
function importChannelRegistrations(cb) {
    var insert = "INSERT INTO `channels` (`name`, `owner`, `time`) VALUES (?, ?, ?)";

    queryOld("SELECT * FROM channels", function (err, rows) {
        if (err) {
            cb(err);
            return;
        }

        rows.sort(function (a, b) {
            return a.id - b.id;
        });

        var aq = new AsyncQueue();
        rows.forEach(function (r) {
            var data = [r.name, r.owner, Date.now()];
            aq.queue(function (lock) {
                queryNew(insert, data, function (err) {
                    if (!err) {
                        console.log("Imported channel record " + r.name + " (" + r.owner + ")");
                    }
                    lock.release();
                });
            });
        });

        aq.queue(function (lock) {
            lock.release();
            cb();
        });
    });
}

/**
 * Imports ranks/bans/library
 */
function importChannelTables(cb) {
    console.log("Importing channel ranks, libraries, bans");
    queryOld("SELECT * FROM `channels`", function (err, rows) {
        if (err) {
            cb(err);
            return;
        }

        var aq = new AsyncQueue();

        rows.forEach(function (r) {
            aq.queue(function (lock) {
                console.log("Creating channel tables for "+r.name);
                tables.createChannelTables(r.name, queryNew, function () {
                    copyChannelTables(r.name, function () {
                        lock.release();
                    });
                });
            });
        });

        aq.queue(function (lock) {
            lock.release();
            cb();
        });
    });
}

function copyChannelTables(name, cb) {
    var copyRanks = function () {
        queryOld("SELECT * FROM `chan_"+name+"_ranks`", function (err, rows) {
            if (err) {
                cb(err);
                return;
            }

            rows = rows.filter(function (r) {
                return r.rank > 1;
            });

            rows = rows.map(function (r) {
                if (r.rank === 10) {
                    r.rank = 5;
                } else if (r.rank >= 3 && r.rank < 10) {
                    r.rank = 4;
                }
                return [r.name, r.rank];
            });

            if (rows.length === 0) {
                console.log("`chan_"+name+"_ranks` is empty");
                copyLibrary();
                return;
            }

            console.log("Copying `chan_"+name+"_ranks`");
            queryNew("INSERT INTO `chan_"+name+"_ranks` VALUES ?", [rows], copyLibrary);
        });
    };

    var copyLibrary = function () {
        queryOld("SELECT * FROM `chan_"+name+"_library`", function (err, rows) {
            if (err) {
                cb(err);
                return;
            }

            rows = rows.map(function (r) {
                return [r.id, r.title, r.seconds, r.type];
            });

            if (rows.length === 0) {
                console.log("`chan_"+name+"_library` is empty");
                copyBans();
                return;
            }

            var subs = [];
            while (rows.length > 1000) {
                subs.push(rows.slice(0, 1000));
                rows = rows.slice(1000);
            }

            if (rows.length > 0) {
                subs.push(rows);
            }

            if (subs.length > 1) {
                console.log("`chan_"+name+"_library` is >1000 rows, requires multiple inserts");
            }

            var aq = new AsyncQueue();
            subs.forEach(function (s) {
                aq.queue(function (lock) {
                    console.log("Copying `chan_"+name+"_library`");
                    queryNew("INSERT INTO `chan_"+name+"_library` VALUES ?",
                             [s], function () {
                        lock.release();
                    });
                });
            });

            aq.queue(function (lock) {
                lock.release();
                copyBans();
            });
        });
    };

    var copyBans = function () {
        queryOld("SELECT * FROM `chan_"+name+"_bans`", function (err, rows) {
            if (err) {
                cb(err);
                return;
            }

            rows = rows.map(function (r) {
                return [r.id, r.ip, r.name, r.bannedby, r.reason];
            });

            if (rows.length === 0) {
                console.log("`chan_"+name+"_bans` is empty");
                cb();
                return;
            }

            console.log("Copying `chan_"+name+"_bans`");
            queryNew("INSERT INTO `chan_"+name+"_bans` VALUES ?", [rows], cb);
        });
    };

    copyRanks();
}

function importGlobalBans(cb) {
    console.log("Importing global bans");
    queryOld("SELECT * FROM `global_bans`", function (err, bans) {
        if (err) {
            cb(err);
            return;
        }

        bans = bans.map(function (b) {
            return [b.ip, b.reason];
        });
        queryNew("INSERT INTO `global_bans` VALUES ?", [bans], cb);
    });
}

function importUserPlaylists(cb) {
    console.log("Importing user playlists");
    queryOld("SELECT * FROM `user_playlists`", function (err, pls) {
        if (err) {
            cb(err);
            return;
        }

        pls = pls.map(function (pl) {
            return [pl.user, pl.name, pl.contents, pl.count, pl.duration];
        });
        var subs = [];
        while (pls.length > 10) {
            subs.push(pls.slice(0, 10));
            pls = pls.slice(10);
        }

        if (pls.length > 0) {
            subs.push(pls);
        }

        var aq = new AsyncQueue();
        subs.forEach(function (s) {
            aq.queue(function (lock) {
                queryNew("INSERT INTO `user_playlists` VALUES ?", [s], function () {
                    lock.release();
                });
            });
        });

        aq.queue(function (lock) {
            lock.release();
            cb();
        });
    });
}

function importAliases(cb) {
    console.log("Importing aliases");
    queryOld("SELECT * FROM `aliases`", function (err, aliases) {
        if (err) {
            cb(err);
            return;
        }

        aliases = aliases.map(function (al) {
            return [al.visit_id, al.ip, al.name, al.time];
        });

        var subs = [];
        while (aliases.length > 1000) {
            subs.push(aliases.slice(0, 1000));
            aliases = aliases.slice(1000);
        }

        if (aliases.length > 0) {
            subs.push(aliases);
        }

        var aq = new AsyncQueue();
        subs.forEach(function (s) {
            aq.queue(function (lock) {
                queryNew("INSERT INTO `aliases` VALUES ?", [s], function () {
                    lock.release();
                });
            });
        });

        aq.queue(function (lock) {
            lock.release();
            cb();
        });
    });
}

function main() {
    var aq = new AsyncQueue();
    var readline = require("readline");
    console.log("This script will generate a lot of text output, both informational and " +
                "possibly errors.  I recommend running it as `node import.js | " +
                "tee import.log` or similar to pipe output to a log file for easy reading");
    var rl = readline.createInterface({
        input: process.stdin,
        output: process.stdout
    });
    aq.queue(function (lock) {
        rl.question("2.x host: ", function (host) {
            olddb.host = host;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        rl.question("2.x username: ", function (user) {
            olddb.user = user;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        rl.question("2.x password: ", function (pw) {
            olddb.password = pw;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        rl.question("2.x database: ", function (db) {
            olddb.database = db;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        rl.question("3.0 host: ", function (host) {
            newdb.host = host;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        rl.question("3.0 username: ", function (user) {
            newdb.user = user;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        rl.question("3.0 password: ", function (pw) {
            newdb.password = pw;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        rl.question("3.0 database: ", function (db) {
            newdb.database = db;
            lock.release();
        });
    });
    aq.queue(function (lock) {
        oldpool = mysql.createPool(olddb);
        newpool = mysql.createPool(newdb);
        queryOld = query.bind(this, oldpool);
        queryNew = query.bind(this, newpool);
        startImport();
    });
}

function startImport() {
    tables.init(queryNew, function (err) {
        if (!err) {
            var aq = new AsyncQueue();
            aq.queue(function (lock) {
                importUsers(function () {
                    lock.release();
                });
            });
            aq.queue(function (lock) {
                importChannelRegistrations(function () {
                    lock.release(); });
            });
            aq.queue(function (lock) {
                importChannelTables(function () {
                    lock.release();
                });
            });
            aq.queue(function (lock) {
                importGlobalBans(function () {
                    lock.release();
                });
            });
            aq.queue(function (lock) {
                importUserPlaylists(function () {
                    lock.release();
                });
            });
            aq.queue(function (lock) {
                importAliases(function () {
                    lock.release();
                });
            });
            aq.queue(function (lock) {
                process.exit(0);
            });
        } else {
            console.log("[ERROR] Aborting due to errors initializing tables");
        }
    });
}

main();