Commit b1af9716 authored by Marco Flowers's avatar Marco Flowers

new database schema, json api and website updated

parent 762febc5
TODO:
- Add the rest of the url checks to yp-cgi.js, also might turn the checks into an
admin interface
- Add meaningful error codes to the json api, not just returning empty []
- Add more options to the api if needed (Grouping of streams by Channel, geographical area)
https://trac.xiph.org/ticket/1958
- Create documenation for the json api
- Finish the html directory site
- Add caching to the json api ? maybe
......@@ -5,17 +5,24 @@ var express = require('express'),
cache_manager = require('cache-manager'),
bodyParser = require('body-parser'),
qs = require('qs'),
conf = require('konphyg')(__dirname + '/config');
conf = require('konphyg')(__dirname + '/config'),
validator = require('validator');
var cache = cache_manager.caching({store: "memory", max: 100, ttl: 10});
var app = express();
var config = conf('config');
/* Controllers */
var index = require('./controllers/index.js')(query, cache);
var genres = require('./controllers/genres.js')(query, cache);
var formats = require('./controllers/formats.js')(query, cache);
var yp_cgi = require('./controllers/yp-cgi.js')(query, qs);
var streamApi = require('./controllers/stream-api.js')(query, cache);
var index = require('./controllers/index.js')(query, cache, streamApi);
var genres = require('./controllers/genres.js')(query, cache, streamApi);
var formats = require('./controllers/formats.js')(query, cache, streamApi);
var yp_cgi = require('./controllers/yp-cgi.js')(query, qs, validator);
var listen = require('./controllers/listen.js')(query, qs, streamApi)
query.connectionParameters = config.db;
......@@ -31,11 +38,60 @@ app.engine('html', swig.renderFile);
app.set('view engine', 'html');
app.set('views', __dirname + '/views');
/* Routes */
/* Website Routes */
app.get('/', index);
app.post('/cgi-bin/yp-cgi', yp_cgi);
app.get('/by_genre/:genre', genres);
app.get('/by_format/:format', formats);
app.post('/cgi-bin/yp-cgi', yp_cgi);
app.get('/listen/:streamId/:filename',listen)
/* JSON API */
function respond(res, err, rows, result)
{
if(err)
{
res.send([]);
}
else
{
res.send(rows);
}
}
app.get('/streams/', function(req,res){
res.set('Content-Type', 'application/json')
streamApi(req.query,function(err, rows,result){
respond(res,err,rows,result)
});
});
app.get('/streams/:streamId',function(req,res){
res.set('Content-Type', 'application/json')
params = {'id':req.params.streamId}
streamApi(params,function(err,rows,result){
if(err || result.rowCount != 1)
{
res.send([]);
}
else
{
res.send(rows[0]);
}
});
})
app.get('/genres/', function(req, res) {
res.set('Content-Type', 'application/json')
var genresq = 'SELECT DISTINCT val FROM (SELECT unnest(genres) as val FROM streams) s;';
query(genresq, function(err, rows, result) {
respond(res,err,rows,result)
});
});
app.get('/formats/', function(req, res) {
res.set('Content-Type', 'application/json')
var formats = 'SELECT DISTINCT val FROM (SELECT unnest(codec_sub_types) as val FROM streams) s;';
query(formats, function(err, rows, result) {
respond(res,err,rows,result)
});
});
var server = app.listen(3000, function() {
console.log('Listening on port %d', server.address().port);
......@@ -46,78 +102,9 @@ process.on('SIGINT', function() {
process.exit();
});
/*
var interval = setInterval(deleteOldServers, 240000, 4);
function deleteOldServers(time, cb) {
query('DELETE FROM servers WHERE lasttouch < NOW() - INTERVAL \'' + time.toString() + ' minutes\';', cb);
}
app.get('/view/', function(req, res) {
//console.log("[GET " + req.originalUrl + "]:")
res.set('Content-Type', 'application/json')
if(!req.param('genre')) {
query('SELECT array_to_json(array_agg(row_to_json(servers))) FROM servers;', function(err, rows, result) {
//console.log(err)
res.send(rows[0].array_to_json)
});
} else {
query('SELECT array_to_json(array_agg(row_to_json(servers))) FROM servers WHERE $1 = ANY (genres);',
[req.param('genre')],
function(err, rows, result) {
//console.log(err)
res.send(rows[0].array_to_json)
});
}
});
app.get('/insert/', function(req, res) {
//console.log("[GET " + req.originalUrl + "]:")
query('INSERT INTO servers VALUES (\
uuid_generate_v4(),\
$1,\
$2,\
$3,\
$4,\
$5,\
$6,\
$7,\
$8,\
$9,\
now()\
);', [req.param('sn'), req.param('type'), [req.param('genres')], req.param('b'), req.param('listenurl'),
req.param('cpswd'), req.param('desc'), req.param('url'), [req.param('stype')]], function(err, rows, result) {
//console.log(err)
res.send(err)
});
});
app.get('/genres/', function(req, res) {
var genresq = 'SELECT DISTINCT val FROM (SELECT unnest(genres) as val FROM servers) s;';
query(genresq, function(err, rows, result) {
res.send(rows);
});
});
\ No newline at end of file
*/
{
"db" : "postgres://user:passwd@host:port/dbname"
"db" : "postgres://marcof@localhost/marcof"
}
\ No newline at end of file
var query, cache;
var query, cache, streamApi;
function init(q, c) {
function init(q, c, s) {
query = q;
cache = c;
streamApi = s;
return byFormat;
}
......@@ -33,11 +34,10 @@ function getCachedFormatStreams(format, cb) {
}
function getFormatStreams(format, cb) {
query("SELECT id, server_name, server_type, genres, bitrate, listenurl, description, url, \
codec_sub_types, songname, listeners FROM servers WHERE $1 = ANY (codec_sub_types);", [format],
function(err, rows, result) {
params = {'format':format}
streamApi(params,function(err, rows, result) {
cb(err, result);
});
}
module.exports = init;
\ No newline at end of file
module.exports = init;
var query, cache;
var query, cache, streamApi;
function init(q, c) {
function init(q, c, s) {
query = q;
cache = c;
streamApi = s;
return byGenre;
}
......@@ -33,11 +34,10 @@ function getCachedGenreStreams(genre, cb) {
}
function getGenreStreams(genre, cb) {
query("SELECT id, server_name, server_type, genres, bitrate, listenurl, description, url, \
codec_sub_types, songname, listeners FROM servers WHERE $1 = ANY (genres);", [genre],
function(err, rows, result) {
params = {'genre':genre}
streamApi(params,function(err, rows, result) {
cb(err, result);
});
}
module.exports = init;
\ No newline at end of file
module.exports = init;
var query, cache;
var query, cache, streamApi;
function init(q, c) {
function init(q, c, s) {
query = q;
cache = c;
streamApi = s;
return index;
}
......@@ -17,6 +18,7 @@ function index(req, res) {
} else {
error = false;
}
console.log(result.rows)
res.render("index", {
title: '',
servers: result.rows,
......@@ -33,11 +35,10 @@ function getCachedRandomStreams(count, cb) {
}
function getRandomStreams(count, cb) {
query('SELECT id, server_name, server_type, genres, bitrate, listenurl, description, url, \
codec_sub_types, songname, listeners FROM servers ORDER BY random() LIMIT $1;', [count],
function(err, rows, result) {
params = {'limit':count,'order':0}
streamApi(params,function(err, rows, result) {
cb(err, result);
});
}
module.exports = init;
\ No newline at end of file
module.exports = init;
var query, cache, streamApi;
function init(q, c, s) {
query = q;
cache = c;
streamApi = s;
return getListen;
}
function getListen(req, res) {
params = {'id':req.params.streamId}
console.log(req.get('User-Agent'))
var userAgent = req.get('User-Agent')
filename = req.params.filename
streamApi(params,function(err, rows, result) {
if(err || result.rowCount != 1)
{
res.status(404);
res.send('Stream not in the Database');
return
}
var extension = filename.substr(filename.lastIndexOf('.')+1)
console.log(extension)
if(extension != 'm3u' && extension != 'xspf')
{
res.status(400);
res.send('Not the right extension');
return
}
var listenurls = rows[0].listenurls
if(extension == 'm3u')
{
var outputString = ''
for (var i = 0; i < listenurls.length; i++)
{
outputString += listenurls[i]+'\r\n'
}
res.set('Content-Type','audio/x-mpegurl')
if(userAgent.indexOf('/MSIE 5.5/') != -1)
{
res.set('Content-Disposition','inline; filename="listen.m3u"')
}
else
{
res.set('Content-Disposition','filename="listen.m3u"')
}
res.send(outputString)
}
else if(extension == 'xspf')
{
var outputString = '<?xml version="1.0" encoding="UTF-8"?>\
\r\n<playlist version="1" xmlns="http://xspf.org/ns/0/">\
\r\n <title>'+rows[0].stream_name+'</title>\
\r\n <info>'+rows[0].url+'</info>\
\r\n <trackList>'
for (var i = 0; i < listenurls.length; i++)
{
outputString +='\r\n <track><location>'+listenurls[i]+'</location></track>'
}
outputString +='\r\n </trackList>\r\n</playlist>'
res.set('Content-Type','application/xspf+xml')
res.set('Content-Disposition','filename="listen.xspf"')
if(userAgent.indexOf('/MSIE 5.5/') != -1)
{
res.set('Content-Disposition','inline; filename="listen.xspf"')
}
else
{
res.set('Content-Disposition','filename="listen.xspf"')
}
res.send(outputString)
}
});
}
module.exports = init;
var query, cache;
function init(q, c) {
query = q;
cache = c;
return streamApi;
}
/*
Takes in paramters {}
q = Search string
id, limit, offset, genre, format, order(0 Random, 1 Listeners Desc, 2 Listeners Asc)
callback is called during the resulting query, should be
function(err, rows, result){}
Turns parameters into a sql query on streams
*/
function streamApi(params,callback)
{
var allStreamsBeg = 'SELECT s.id, s.stream_name, s.stream_type, s.description,s.songname, s.url, s.avg_listening_time, s.codec_sub_types, s.bitrate, s.hits, s.cm, s.samplerate, s.channels, s.quality, s.genres, array_agg(sm.listenurl) AS listenurls, COUNT(sm.listeners) AS listeners, COUNT(sm.max_listeners) AS max_listeners FROM streams s INNER JOIN server_mounts AS sm ON s.id = sm.stream_id '
var allStreamsGroup = 'GROUP BY s.id '
var countItems = 1
var items = []
var queryString = allStreamsBeg
var genre, format, streamId, search, order, limit, offset;
if(params.genre)
{
genre = params.genre
if(Array.isArray(genre))
{
for(var i =0; i <genre.length;i++)
{
queryString += genreFormatModifier(countItems, 's.genres')
countItems++
items.push(genre[i])
}
}
else
{
queryString += genreFormatModifier(countItems, 's.genres')
countItems++
items.push(genre)
}
}
if(params.format)
{
format = params.format
if(Array.isArray(format))
{
for(var i =0; i <genre.length;i++)
{
queryString += genreFormatModifier(countItems, 's.codec_sub_types')
countItems++
items.push(format[i])
}
}
else
{
queryString += genreFormatModifier(countItems, 's.codec_sub_types')
countItems++
items.push(format)
}
}
if(params.id)
{
streamId = params.id
if(countItems > 1)
{
queryString += 'AND $'+countItems+' = s.id '
}
else
{
queryString += 'WHERE $'+countItems+' = s.id '
}
countItems++
items.push(streamId)
}
if(params.q)
{
search = params.q
var add = false;
var searchWord = '';
var searchArray ='(';
if(Array.isArray(search))
{
search = search[0]
}
search = search.split(' ')
var searchString = 'WHERE ('
if(countItems > 1)
{
searchString = 'AND ('
}
for(var i=0;i<search.length;i++)
{
if(i>3)
{
break;
}
// minimum word length of 3
if(search[i].length<3)
{
continue;
}
add = true
//maximum word length of 10
if(search[i].length>10)
{
search[i] = search[i].substring(0,10)
}
searchWord += search[i]+'%'
searchString += '$'+countItems+' LIKE ANY(s.genres) OR '
countItems++
items.push(search[i])
}
if(add)
{
searchString += 's.stream_name LIKE $'+countItems
+' OR s.description LIKE $'+countItems+' '
items.push(searchWord)
countItems++
searchString += ') '
queryString += searchString
}
else
{
// search query won't return any results
callback(0,[],{})
return
}
}
// add in the Group BY
queryString = queryString + allStreamsGroup
if(params.order)
{
order = params.order
if(order == 0)
{
queryString += 'ORDER BY random() '
}
else if(order == 1)
{
queryString += 'ORDER BY listeners DESC'
}
else if(order == 2)
{
queryString += 'ORDER BY listeners ASC'
}
}
if(params.limit) {
limit = params.limit
if(Array.isArray(limit))
{
limit = limit[0]
}
else
{
queryString += 'LIMIT $'+countItems+' '
countItems++
items.push(limit)
}
}
if(params.offset) {
offset = params.offset
if(Array.isArray(offset))
{
offset = offset[0]
}
else
{
queryString += 'OFFSET $'+countItems+' '
countItems++
items.push(offset)
}
}
//console.log(queryString)
query(queryString,items,callback)
}
function genreFormatModifier(count, type)
{
if(count > 1)
{
return ('AND $'+count+' = ANY('+type+') ')
}
else
{
return ('WHERE $'+count+' = ANY('+type+') ')
}
}
module.exports = init;
var query, qs;
var query, qs, validator;
var async = require('async');
function init(q, q_) {
function init(q, q_, v) {
query = q;
qs = q_;
validator = v;
return dispatcher;
}
......@@ -10,72 +12,214 @@ function dispatcher(req, res) {
if(req.body.action == "add") {
ypAdd(req, res);
} else if(req.body.action == "touch") {
ypTouch(req, res);
ypTouch(req, res);
} else if (req.body.action == "remove") {
ypRemove(req, res);
}
}
function checkPresent(toCheck, check)
{
for(var i=0;i<check.length;i++)
{
if(!(check[i] in toCheck))
{
return false;
}
}
return true;
}
function multiIndexOf(toCheck, check)
{
for(var i=0;i<check.length;i++)
{
if(toCheck.indexOf(check[i]) != -1)
{
return false;
}
}
return true;
}
function ypAdd(req, res) {
if (req.body.listenurl.indexOf("http://localhost") != -1 ||
req.body.listenurl.indexOf("https://localhost") != -1) {
ypRes(res, false, "Your hostname is localhost, this is an indicator of a misconfigured server", -1, null);
var start = new Date().getTime();
var params = req.body;
// check mandatory arguments
var mandatoryArgs = ['sn', 'type', 'genre', 'listenurl']
var illegalListenUrls = ['dev.local','testvm.hivane.net',
'backup.abidingradio.com']
var abuseIps = ['92.246.30.112']
var misconfiguredUrls = ['hostingcenter.com']
var misconfiguredIps = ['192.240.97.','192.240.102.','50.7..']
//need to add others
var defaultServerNames = ['Unspecified name','This is my server name',
'Stream Name','My Station name']
if( checkPresent(params, mandatoryArgs) == false)
{
ypRes(res, false, "Not enough arguments", -1, null);
return;
}
if( validator.isURL(params.listenurl) == false)
{
ypRes(res, false, "Not a real listenurl", -1, null);
return;
}
if( multiIndexOf(params.listenurl, illegalListenUrls) == false)
{
ypRes(res, false, "Illegal listen_url. Don't test against a production \
server, thanks! ", -1, null);
return;
}
if (req.body.sn === 'Unspecified name') {
ypRes(res, false, "You have to specify a name for your stream.", -1, null);
if( multiIndexOf(params.listenurl, abuseIps) == false)
{
ypRes(res, false, "Your server has been banned for abuse, have a nice \
day!", -1, null);
return;
}
if( multiIndexOf(params.listenurl, misconfiguredUrls) == false)// || multiIndexOf(ip,misconfiguredIps))
{
ypRes(res, false, "The network range in which your server resides has \
been suspended due to a high number of wrongly configured servers! Please\
contact webmaster@xiph.org urgently!", -1, null);
return;
}
if( multiIndexOf(params.sn, defaultServerNames) == false)
{
ypRes(res, false, "Default stream name detected, please configure your \
source client, thanks! ", -1, null);
return;
}
//parse the body
var final = parseBody(req.body);
var insert = 'INSERT INTO servers (id, lasttouch, server_name, server_type, genres, bitrate, listenurl, \
cluster_pass, description, url, codec_sub_types, channels, samplerate, quality) \
VALUES (uuid_generate_v4(), now(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \
RETURNING id;'
query(insert, [final.server_name, final.server_type, final.genres, final.bitrate,
final.listenurl, final.cluster_pass, final.description, final.url,
final.codec_sub_types, final.channels, final.samplerate, final.quality],
function(err, rows, result) {
console.log(err);
if (err) {
ypRes(res, false, "Error adding the stream", -1, null);
} else {
ypRes(res, true, "Successfully Added", rows[0].id, 200);
};
var insertStream = 'INSERT INTO streams (stream_name, stream_type, genres, bitrate, cluster_pass, description, url, codec_sub_types, channels, samplerate, quality) \
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) \
RETURNING id;'
var insertServerMount = 'INSERT INTO server_mounts (sid, stream_id, lasttouch, listenurl) \
VALUES (uuid_generate_v4(), $1, now(), $2) \