specter/CyTube
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Change charset for certain fields to utf8mb4

Browse source 
The underlying cause of #419 is the default utf8 collation in MySQL/MariaDB, which only supports the base plane of Unicode (\u0000-\uffff).  By changing the collation to utf8mb4_general_ci, stuff like ban reasons and profile text may have emoji and other non-base-plane Unicode.

The charset for playlist titles is NOT changed, and non-base-plane characters are replaced by question marks.  This is because switching to utf8mb4 would make the primary key too long.
This commit is contained in:
Calvin Montgomery 2014-12-14 21:33:48 -05:00
parent 4b8681c2a4
commit 9deff9bdb1
3 changed files with 8 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/channel/playlist.js
View file

@ -1243,6 +1243,8 @@ PlaylistModule.prototype.handleClonePlaylist = function (user, data) {
}); });
} }
data.name = data.name.replace(/[^\u0000-\uffff]/g, "?");
var pl = this.items.toArray(); var pl = this.items.toArray();
var self = this; var self = this;
db.saveUserPlaylist(pl, user.getName(), data.name, function (err) { db.saveUserPlaylist(pl, user.getName(), data.name, function (err) {

3
lib/database.js
View file

@ -17,7 +17,8 @@ module.exports.init = function () {
user: Config.get("mysql.user"), user: Config.get("mysql.user"),
password: Config.get("mysql.password"), password: Config.get("mysql.password"),
database: Config.get("mysql.database"), database: Config.get("mysql.database"),
multipleStatements: true multipleStatements: true,
charset: "UTF8MB4_GENERAL_CI" // Needed for emoji and other non-BMP unicode
}); });
// Test the connection // Test the connection

8
lib/database/tables.js
View file

@ -5,7 +5,7 @@ const TBL_USERS = "" +
"`password` VARCHAR(64) NOT NULL," + "`password` VARCHAR(64) NOT NULL," +
"`global_rank` INT NOT NULL," + "`global_rank` INT NOT NULL," +
"`email` VARCHAR(255) NOT NULL," + "`email` VARCHAR(255) NOT NULL," +
"`profile` TEXT NOT NULL," + "`profile` TEXT CHARACTER SET utf8mb4 NOT NULL," +
"`ip` VARCHAR(39) NOT NULL," + "`time` BIGINT NOT NULL," + "`ip` VARCHAR(39) NOT NULL," + "`time` BIGINT NOT NULL," +
"PRIMARY KEY(`id`)," + "PRIMARY KEY(`id`)," +
"UNIQUE(`name`)) " + "UNIQUE(`name`)) " +
@ -23,7 +23,7 @@ const TBL_CHANNELS = "" +
const TBL_GLOBAL_BANS = "" + const TBL_GLOBAL_BANS = "" +
"CREATE TABLE IF NOT EXISTS `global_bans` (" + "CREATE TABLE IF NOT EXISTS `global_bans` (" +
"`ip` VARCHAR(39) NOT NULL," + "`ip` VARCHAR(39) NOT NULL," +
"`reason` VARCHAR(255) NOT NULL," + "`reason` VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL," +
"PRIMARY KEY (`ip`)) " + "PRIMARY KEY (`ip`)) " +
"CHARACTER SET utf8"; "CHARACTER SET utf8";
@ -75,7 +75,7 @@ const TBL_META = "" +
const TBL_LIBRARIES = "" + const TBL_LIBRARIES = "" +
"CREATE TABLE IF NOT EXISTS `channel_libraries` (" + "CREATE TABLE IF NOT EXISTS `channel_libraries` (" +
"`id` VARCHAR(255) NOT NULL," + "`id` VARCHAR(255) NOT NULL," +
"`title` VARCHAR(255) NOT NULL," + "`title` VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL," +
"`seconds` INT NOT NULL," + "`seconds` INT NOT NULL," +
"`type` VARCHAR(2) NOT NULL," + "`type` VARCHAR(2) NOT NULL," +
"`meta` TEXT NOT NULL," + "`meta` TEXT NOT NULL," +
@ -97,7 +97,7 @@ const TBL_BANS = "" +
"`ip` VARCHAR(39) NOT NULL," + "`ip` VARCHAR(39) NOT NULL," +
"`name` VARCHAR(20) NOT NULL," + "`name` VARCHAR(20) NOT NULL," +
"`bannedby` VARCHAR(20) NOT NULL," + "`bannedby` VARCHAR(20) NOT NULL," +
"`reason` VARCHAR(255) NOT NULL," + "`reason` VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL," +
"`channel` VARCHAR(30) NOT NULL," + "`channel` VARCHAR(30) NOT NULL," +
"PRIMARY KEY (`id`, `channel`), UNIQUE (`name`, `ip`, `channel`), " + "PRIMARY KEY (`id`, `channel`), UNIQUE (`name`, `ip`, `channel`), " +
"INDEX (`ip`, `channel`), INDEX (`name`, `channel`)" + "INDEX (`ip`, `channel`), INDEX (`name`, `channel`)" +