Add YouTube v3 API

YouTube v2 is still supported as a fallback, but will log a warning
message to the error log as v2 is expected to be closed shortly after
April 20, 2015.

See also:
http://youtube-eng.blogspot.com/2015/03/dude-are-you-still-on-youtube-api-v2.html
This commit is contained in:
calzoneman 2015-03-27 18:44:46 -05:00
parent 9541b40f68
commit 27a50cb702
6 changed files with 310 additions and 225 deletions

View file

@ -114,8 +114,11 @@ mail:
from-address: 'some.user@gmail.com' from-address: 'some.user@gmail.com'
from-name: 'CyTube Services' from-name: 'CyTube Services'
# GData API v2 developer key (for non-anonymous youtube requests) # YouTube v3 API key
youtube-v2-key: '' # See https://developers.google.com/youtube/registering_an_application
# Google is closing the v2 API (which allowed anonymous requests) on
# April 20, 2015 so you must register a v3 API key now.
youtube-v3-key: ''
# Minutes between saving channel state to disk # Minutes between saving channel state to disk
channel-save-interval: 5 channel-save-interval: 5
# Limit for the number of channels a user can register # Limit for the number of channels a user can register

View file

@ -67,7 +67,7 @@ LibraryModule.prototype.handleUncache = function (user, data) {
LibraryModule.prototype.handleSearchMedia = function (user, data) { LibraryModule.prototype.handleSearchMedia = function (user, data) {
var query = data.query.substring(0, 100); var query = data.query.substring(0, 100);
var searchYT = function () { var searchYT = function () {
InfoGetter.Getters.ytSearch(query.split(" "), function (e, vids) { InfoGetter.Getters.ytSearch(query, function (e, vids) {
if (!e) { if (!e) {
user.socket.emit("searchResults", { user.socket.emit("searchResults", {
source: "yt", source: "yt",

View file

@ -509,6 +509,7 @@ PlaylistModule.prototype.queueYouTubePlaylist = function (user, data) {
self.channel.activeLock.lock(); self.channel.activeLock.lock();
vids.forEach(function (media) { vids.forEach(function (media) {
data.link = util.formatLink(media.id, media.type);
self._addItem(media, data, user); self._addItem(media, data, user);
}); });
self.channel.activeLock.release(); self.channel.activeLock.release();

View file

@ -58,7 +58,7 @@ var defaults = {
"from-address": "some.user@gmail.com", "from-address": "some.user@gmail.com",
"from-name": "CyTube Services" "from-name": "CyTube Services"
}, },
"youtube-v2-key": "", "youtube-v3-key": "",
"channel-save-interval": 5, "channel-save-interval": 5,
"max-channels-per-user": 5, "max-channels-per-user": 5,
"max-accounts-per-ip": 5, "max-accounts-per-ip": 5,
@ -346,6 +346,17 @@ function preprocessConfig(cfg) {
cfg["link-domain-blacklist-regex"] = new RegExp("$^", "gi"); cfg["link-domain-blacklist-regex"] = new RegExp("$^", "gi");
} }
if (cfg["youtube-v3-key"]) {
require("cytube-mediaquery/lib/provider/youtube").setApiKey(
cfg["youtube-v3-key"]);
} else {
Logger.errlog.log("Warning: No YouTube v3 API key set. YouTube lookups will " +
"fall back to the v2 API, which is scheduled for closure soon after " +
"April 20, 2015. See " +
"https://developers.google.com/youtube/registering_an_application for " +
"information on registering an API key.");
}
return cfg; return cfg;
} }

View file

@ -7,6 +7,8 @@ var CustomEmbedFilter = require("./customembed").filter;
var Server = require("./server"); var Server = require("./server");
var Config = require("./config"); var Config = require("./config");
var ffmpeg = require("./ffmpeg"); var ffmpeg = require("./ffmpeg");
require("cytube-mediaquery"); // Initialize sourcemaps
var YouTube = require("cytube-mediaquery/lib/provider/youtube");
/* /*
* Preference map of quality => youtube formats. * Preference map of quality => youtube formats.
@ -63,249 +65,68 @@ var urlRetrieve = function (transport, options, callback) {
var Getters = { var Getters = {
/* youtube.com */ /* youtube.com */
yt: function (id, callback) { yt: function (id, callback) {
var sv = Server.getServer(); if (!Config.get("youtube-v3-key")) {
return Getters.yt2(id, callback);
var m = id.match(/([\w-]{11})/);
if (m) {
id = m[1];
} else {
callback("Invalid ID", null);
return;
} }
var options = {
host: "gdata.youtube.com",
port: 443,
path: "/feeds/api/videos/" + id + "?v=2&alt=json",
method: "GET",
dataType: "jsonp",
timeout: 1000
};
if (Config.get("youtube-v2-key")) { YouTube.lookup(id).then(function (video) {
options.headers = {
"X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
};
}
urlRetrieve(https, options, function (status, data) {
switch (status) {
case 200:
break; /* Request is OK, skip to handling data */
case 400:
return callback("Invalid request", null);
case 403:
return callback("Private video", null);
case 404:
return callback("Video not found", null);
case 500:
case 503:
return callback("Service unavailable", null);
default:
return callback("HTTP " + status, null);
}
var buffer = data;
try {
data = JSON.parse(data);
/* Check for embedding restrictions */
if (data.entry.yt$accessControl) {
var ac = data.entry.yt$accessControl;
for (var i = 0; i < ac.length; i++) {
if (ac[i].action === "embed") {
if (ac[i].permission === "denied") {
callback("Embedding disabled", null);
return;
}
break;
}
}
}
var seconds = data.entry.media$group.yt$duration.seconds;
var title = data.entry.title.$t;
var meta = {}; var meta = {};
/* Check for country restrictions */ if (video.meta.blocked) {
if (data.entry.media$group.media$restriction) { meta.restricted = video.meta.blocked;
var rest = data.entry.media$group.media$restriction;
if (rest.length > 0) {
if (rest[0].relationship === "deny") {
meta.restricted = rest[0].$t;
} }
}
} var media = new Media(video.id, video.title, video.duration, "yt", meta);
var media = new Media(id, title, seconds, "yt", meta);
callback(false, media); callback(false, media);
} catch (e) { }).catch(function (err) {
// Gdata version 2 has the rather silly habit of callback(err.message, null);
// returning error codes in XML when I explicitly asked
// for JSON
var m = buffer.match(/<internalReason>([^<]+)<\/internalReason>/);
if (m === null)
m = buffer.match(/<code>([^<]+)<\/code>/);
var err = e;
if (m) {
if(m[1] === "too_many_recent_calls") {
err = "YouTube is throttling the server right "+
"now for making too many requests. "+
"Please try again in a moment.";
} else {
err = m[1];
}
}
callback(err, null);
}
}); });
}, },
/* youtube.com playlists */ /* youtube.com playlists */
yp: function (id, callback, url) { yp: function (id, callback) {
/** if (!Config.get("youtube-v3-key")) {
* NOTE: callback may be called multiple times, once for each <= 25 video return Getters.yp2(id, callback);
* batch of videos in the list. It will be called in order.
*/
var m = id.match(/([\w-]+)/);
if (m) {
id = m[1];
} else {
callback("Invalid ID", null);
return;
}
var path = "/feeds/api/playlists/" + id + "?v=2&alt=json";
/**
* NOTE: the third parameter, url, is used to chain this retriever
* multiple times to get all the videos from a playlist, as each
* request only returns 25 videos.
*/
if (url !== undefined) {
path = "/" + url.split("gdata.youtube.com")[1];
} }
var options = { YouTube.lookupPlaylist(id).then(function (videos) {
host: "gdata.youtube.com", videos = videos.map(function (video) {
port: 443, var meta = {};
path: path, if (video.meta.blocked) {
method: "GET", meta.restricted = video.meta.blocked;
dataType: "jsonp",
timeout: 1000
};
if (Config.get("youtube-v2-key")) {
options.headers = {
"X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
};
} }
urlRetrieve(https, options, function (status, data) { return new Media(video.id, video.title, video.duration, "yt", meta);
switch (status) { });
case 200:
break; /* Request is OK, skip to handling data */
case 400:
return callback("Invalid request", null);
case 403:
return callback("Private playlist", null);
case 404:
return callback("Playlist not found", null);
case 500:
case 503:
return callback("Service unavailable", null);
default:
return callback("HTTP " + status, null);
}
try {
data = JSON.parse(data);
var vids = [];
for(var i in data.feed.entry) {
try {
/**
* FIXME: This should probably check for embed restrictions
* and country restrictions on each video in the list
*/
var item = data.feed.entry[i];
var id = item.media$group.yt$videoid.$t;
var title = item.title.$t;
var seconds = item.media$group.yt$duration.seconds;
var media = new Media(id, title, seconds, "yt");
vids.push(media);
} catch(e) {
}
}
callback(false, vids);
var links = data.feed.link;
for (var i in links) {
if (links[i].rel === "next") {
/* Look up the next batch of videos from the list */
Getters["yp"](id, callback, links[i].href);
}
}
} catch (e) {
callback(e, null);
}
callback(null, videos);
}).catch(function (err) {
callback(err.message, null);
}); });
}, },
/* youtube.com search */ /* youtube.com search */
ytSearch: function (terms, callback) { ytSearch: function (query, callback) {
/** if (!Config.get("youtube-v3-key")) {
* terms is a list of words from the search query. Each word must be return Getters.ytSearch2(query.split(" "), callback);
* encoded properly for use in the request URI
*/
for (var i in terms) {
terms[i] = encodeURIComponent(terms[i]);
}
var query = terms.join("+");
var options = {
host: "gdata.youtube.com",
port: 443,
path: "/feeds/api/videos/?q=" + query + "&v=2&alt=json",
method: "GET",
dataType: "jsonp",
timeout: 1000
};
if (Config.get("youtube-v2-key")) {
options.headers = {
"X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
};
} }
urlRetrieve(https, options, function (status, data) { YouTube.search(query).then(function (res) {
if (status !== 200) { var videos = res.results;
callback("YouTube search: HTTP " + status, null); videos = videos.map(function (video) {
return; var meta = {};
if (video.meta.blocked) {
meta.restricted = video.meta.blocked;
} }
try { var media = new Media(video.id, video.title, video.duration, "yt", meta);
data = JSON.parse(data); media.thumb = { url: video.meta.thumbnail };
var vids = []; return media;
for(var i in data.feed.entry) { });
try {
/**
* FIXME: This should probably check for embed restrictions
* and country restrictions on each video in the list
*/
var item = data.feed.entry[i];
var id = item.media$group.yt$videoid.$t;
var title = item.title.$t;
var seconds = item.media$group.yt$duration.seconds;
var media = new Media(id, title, seconds, "yt");
media.thumb = item.media$group.media$thumbnail[0];
vids.push(media);
} catch(e) {
}
}
callback(false, vids); callback(null, videos);
} catch(e) { }).catch(function (err) {
callback(e, null); callback(err.message, null);
}
}); });
}, },
@ -905,6 +726,254 @@ var Getters = {
var media = new Media(id, title, "--:--", "hb"); var media = new Media(id, title, "--:--", "hb");
callback(false, media); callback(false, media);
}, },
/* youtube.com - old v2 API */
yt2: function (id, callback) {
var sv = Server.getServer();
var m = id.match(/([\w-]{11})/);
if (m) {
id = m[1];
} else {
callback("Invalid ID", null);
return;
}
var options = {
host: "gdata.youtube.com",
port: 443,
path: "/feeds/api/videos/" + id + "?v=2&alt=json",
method: "GET",
dataType: "jsonp",
timeout: 1000
};
if (Config.get("youtube-v2-key")) {
options.headers = {
"X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
};
}
urlRetrieve(https, options, function (status, data) {
switch (status) {
case 200:
break; /* Request is OK, skip to handling data */
case 400:
return callback("Invalid request", null);
case 403:
return callback("Private video", null);
case 404:
return callback("Video not found", null);
case 500:
case 503:
return callback("Service unavailable", null);
default:
return callback("HTTP " + status, null);
}
var buffer = data;
try {
data = JSON.parse(data);
/* Check for embedding restrictions */
if (data.entry.yt$accessControl) {
var ac = data.entry.yt$accessControl;
for (var i = 0; i < ac.length; i++) {
if (ac[i].action === "embed") {
if (ac[i].permission === "denied") {
callback("Embedding disabled", null);
return;
}
break;
}
}
}
var seconds = data.entry.media$group.yt$duration.seconds;
var title = data.entry.title.$t;
var meta = {};
/* Check for country restrictions */
if (data.entry.media$group.media$restriction) {
var rest = data.entry.media$group.media$restriction;
if (rest.length > 0) {
if (rest[0].relationship === "deny") {
meta.restricted = rest[0].$t;
}
}
}
var media = new Media(id, title, seconds, "yt", meta);
callback(false, media);
} catch (e) {
// Gdata version 2 has the rather silly habit of
// returning error codes in XML when I explicitly asked
// for JSON
var m = buffer.match(/<internalReason>([^<]+)<\/internalReason>/);
if (m === null)
m = buffer.match(/<code>([^<]+)<\/code>/);
var err = e;
if (m) {
if(m[1] === "too_many_recent_calls") {
err = "YouTube is throttling the server right "+
"now for making too many requests. "+
"Please try again in a moment.";
} else {
err = m[1];
}
}
callback(err, null);
}
});
},
/* youtube.com playlists - old v2 api */
yp2: function (id, callback, url) {
/**
* NOTE: callback may be called multiple times, once for each <= 25 video
* batch of videos in the list. It will be called in order.
*/
var m = id.match(/([\w-]+)/);
if (m) {
id = m[1];
} else {
callback("Invalid ID", null);
return;
}
var path = "/feeds/api/playlists/" + id + "?v=2&alt=json";
/**
* NOTE: the third parameter, url, is used to chain this retriever
* multiple times to get all the videos from a playlist, as each
* request only returns 25 videos.
*/
if (url !== undefined) {
path = "/" + url.split("gdata.youtube.com")[1];
}
var options = {
host: "gdata.youtube.com",
port: 443,
path: path,
method: "GET",
dataType: "jsonp",
timeout: 1000
};
if (Config.get("youtube-v2-key")) {
options.headers = {
"X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
};
}
urlRetrieve(https, options, function (status, data) {
switch (status) {
case 200:
break; /* Request is OK, skip to handling data */
case 400:
return callback("Invalid request", null);
case 403:
return callback("Private playlist", null);
case 404:
return callback("Playlist not found", null);
case 500:
case 503:
return callback("Service unavailable", null);
default:
return callback("HTTP " + status, null);
}
try {
data = JSON.parse(data);
var vids = [];
for(var i in data.feed.entry) {
try {
/**
* FIXME: This should probably check for embed restrictions
* and country restrictions on each video in the list
*/
var item = data.feed.entry[i];
var id = item.media$group.yt$videoid.$t;
var title = item.title.$t;
var seconds = item.media$group.yt$duration.seconds;
var media = new Media(id, title, seconds, "yt");
vids.push(media);
} catch(e) {
}
}
callback(false, vids);
var links = data.feed.link;
for (var i in links) {
if (links[i].rel === "next") {
/* Look up the next batch of videos from the list */
Getters["yp2"](id, callback, links[i].href);
}
}
} catch (e) {
callback(e, null);
}
});
},
/* youtube.com search - old v2 api */
ytSearch2: function (terms, callback) {
/**
* terms is a list of words from the search query. Each word must be
* encoded properly for use in the request URI
*/
for (var i in terms) {
terms[i] = encodeURIComponent(terms[i]);
}
var query = terms.join("+");
var options = {
host: "gdata.youtube.com",
port: 443,
path: "/feeds/api/videos/?q=" + query + "&v=2&alt=json",
method: "GET",
dataType: "jsonp",
timeout: 1000
};
if (Config.get("youtube-v2-key")) {
options.headers = {
"X-Gdata-Key": "key=" + Config.get("youtube-v2-key")
};
}
urlRetrieve(https, options, function (status, data) {
if (status !== 200) {
callback("YouTube search: HTTP " + status, null);
return;
}
try {
data = JSON.parse(data);
var vids = [];
for(var i in data.feed.entry) {
try {
/**
* FIXME: This should probably check for embed restrictions
* and country restrictions on each video in the list
*/
var item = data.feed.entry[i];
var id = item.media$group.yt$videoid.$t;
var title = item.title.$t;
var seconds = item.media$group.yt$duration.seconds;
var media = new Media(id, title, seconds, "yt");
media.thumb = item.media$group.media$thumbnail[0];
vids.push(media);
} catch(e) {
}
}
callback(false, vids);
} catch(e) {
callback(e, null);
}
});
},
}; };
/** /**

View file

@ -13,6 +13,7 @@
"compression": "^1.3.0", "compression": "^1.3.0",
"cookie-parser": "^1.3.3", "cookie-parser": "^1.3.3",
"csrf": "^2.0.6", "csrf": "^2.0.6",
"cytube-mediaquery": "git://github.com/CyTube/mediaquery",
"cytubefilters": "git://github.com/calzoneman/cytubefilters#7663b3a9", "cytubefilters": "git://github.com/calzoneman/cytubefilters#7663b3a9",
"express": "^4.11.1", "express": "^4.11.1",
"express-minify": "^0.1.3", "express-minify": "^0.1.3",