Commit 4f6c0fa2 authored by Marco Flowers's avatar Marco Flowers

Refined id pagination, reloadable config, error messages

Id pagination was changed to use listener count. Added a link
to reload the config file while the server is running. Errors
in the api now return error status codes and messages.
parent 1e19f887
......@@ -8,12 +8,12 @@ var express = require('express'),
conf = require('konphyg')(__dirname + '/config'),
validator = require('validator'),
xmlbuilder = require('xmlbuilder');
querystring = require('querystring');
var cache = cache_manager.caching({store: "memory", max: 100, ttl: 10});
var app = express();
var config = conf('config');
var config = conf.all().config;
/* Controllers */
var streamsFindBy = require('./controllers/stream-api.js')(query, cache);
......@@ -48,6 +48,20 @@ app.get('/by_genre/:genre', genres);
app.get('/by_format/:format', formats);
app.get('/listen/:streamId/:filename',listen);
// allows updated ban lists to be reloaded
app.get('/reloadconfig/:password',function(req, res) {
// so not everyone can reload the config file
if(req.params.password == config.bansreloadpassword) {
// clear config and reload it
conf.clear();
config = conf.all().config;
res.send("Config reloaded");
} else {
res.status(401);
res.send("Wrong password");
}
});
/* JSON API */
function respond(res, err, rows, result) {
if(err) {
......@@ -59,11 +73,45 @@ function respond(res, err, rows, result) {
app.get('/streams/', function(req,res){
res.set('Content-Type', 'application/json');
var params = req.query;
streamsFindBy(params.format, params.genre, params.q, params.order, params.limit, params.next, params.prev, 1, function(err, rows){
if(err || rows[0].streams == null) {
res.send([]);
var json = 1;
streamsFindBy(params.format, params.genre, params.q, params.order, params.limit, params.starting_after, params.ending_before, json, function(err, rows){
if(err) {
if(err.responsecode) {
res.status(err.responsecode);
} else {
res.status(400);
}
res.send({"error":err.message});
} else {
res.send(rows[0].streams);
var result = rows[0];
if(result.streams == null) {
result.streams = [];
}
result.data = {};
if(params.limit) {
var starting_after = params.starting_after;
var ending_before = params.ending_before;
var limit = params.limit;
var qstring;
if(result.streams.length == limit) {
// delete the previous values
delete params.starting_after;
delete params.ending_before;
// add in the new values
var last_id = rows[0].streams[result.streams.length-1].id;
params.starting_after = last_id;
qstring = querystring.stringify(params);
result.data.next_url = req.path+'?'+qstring;
delete params.starting_after;
params.ending_before = prev_id;
qstring = querystring.stringify(params);
result.data.prev_url = req.path+'?'+qstring;
}
}
res.send(result);
}
});
});
......@@ -71,6 +119,7 @@ app.get('/streams/:streamId', function(req,res){
res.set('Content-Type', 'application/json');
streamFindById(req.params.streamId, 1, function(err,rows,result){
if(err || rows[0].array_to_json == null) {
res.status(404);
res.send({});
} else {
res.send(rows[0].array_to_json[0]);
......@@ -84,6 +133,7 @@ app.get('/genres/', function(req, res) {
respond(res, err, rows, result);
});
});
app.get('/formats/', function(req, res) {
res.set('Content-Type', 'application/json');
var formats = 'SELECT DISTINCT val FROM (SELECT unnest(codec_sub_types) as val FROM streams) s;';
......
{
"db" : "postgres://marcof@localhost/marcof",
"bans": {
"illegalListen":{
"reason": "Not enough arguments",
......@@ -7,12 +6,40 @@
},
"abuse":{
"reason": "Your server has been banned for abuse, have a nice day!",
"listenIps": ["92.246.30.112"]
"listenIps": ["92.246.30.112","222","333"]
},
"misconfigured":{
"reason": "The network range in which your server resides has been suspended due to a high number of wrongly configured servers! Please contact webmaster@xiph.org urgently!",
"listenUrls": ["hostingcenter.com"],
"listenIps": ["192.240.97.","192.240.102.","50.7.."]
},
"illegalListen2":{
"reason":"Whatever",
"listenUrls":["lazaradio.hu","ice.mwsc.tmt.de","host504.com","bristol.railroadradio.net","www.kolombiaestereo.com","radiowanderbuehne.org","live.raincitystream.com","roubaix.fr.shinsen-radio.com",
"lepesradio.hu","laradiohd.com","s-radio.whyza.net","213.240.254.147","radio.adrenalin.fm","99.93.26.131",
"delux-streams.eu","radio.status.ks.ua","campus.longmusic.com","stream1.getradio.ru",
"servercristianonetwork.com","haiku.it.su.se","floodkik.no-ip.org","anytime-fm.i1234.me",
"108.46.109.251","laveteranacentro.com","prosoftnetwork.com","radioonce.mx","carter.sgc-univ.net",
"se-lkr2.stream.sevenbroadcasting.se","tv7n.com","ohmbrew.com","latinafm.com.ve","streamcenter.pro",
"radioutopia.ondametadona.net","www.sentimientosradio.com","magictodayradio.net","radiohd.com.ve",
"mfa05.mfa.go.th","www.radiompt.com.br","stream.pick-nik.ru","stream.ufostation.ru","salomefm.com",
"str0.zenon-media.com","livehost.com.br","radio.sfa.fathi.eu.org","belfast.fm","localhost.com","ix.clpw.ru",
"truecolorsradio.com","misionera.org.ve","thevibefmstlucia.net","www.radiounivo.com","66.135.42.116",
"laperladeltuy.com.ve","i-stereo.ru","topeslaradio.net","apollo21.cdnstream.com","educativa105.org",
"www.Meghedi.com","live.ultra-byte.ro","vps.ds106rad.io","streaming.rtz.rs","streaming.radioparacin.rs",
"netradiofm.com","195.24.224.77","vaststream.com","www.radio8.de","lafieraipiales.com","stream.martini-multimedia.net",
"sabambufm.com","midght-madness.is-a-rockstar.com","relay1.oxyradio.net","stream.ipv6.frequence3.net",
"janebarcelos.com.br","royal-host.net","live.radioanjangsanabogor.com","eclectica.mx","live.rtr.fm","player.morcegaofm.com.br",
"salyut.scenesat.com","nukmradio.com","osbleianos.com.br","radio2.promodeejay.net","stream1.letsgozik.org",
"tigerstream.k2d.com","novafmms.com.br","ondaradio.com.mx","mysuarafm.com","italodiscohits.com",
"reliastream.com","walsh.g-innova.com.ar","recitation.no-ip.org","services.lwbcast.org","305stream.com",
"www.clubkydz.com","marca.com","radioyan.com","radio.ya.dn.ua","radio.radiozlatousti.rs","radioprovinciafm.com",
"www.growradio.org","radiolatina.com.ar","radiocomunidadargentina.org","radiocfa.com.br","quibario.com",
"marketing.com.sv","larumbos.com","kdx.hobby-site.org","fmmonterrico.com","cooldradio.com",
"mai.com.es","manain.org","positiviemixcafe.com","7la.fm","live.unbandsoundz.org.uk","207.210.120.98",
"140.78.254.133","estudio100.com","frisby.com.co","217.144.56.128","ceomedia.cl",
"electronic-dream.com","radio.3rd.name","radiodurisima.com"]
}
}
}
......@@ -8,32 +8,65 @@ function init(q, c, s) {
}
function byFormat(req, res) {
var params = req.params;
var genre = undefined;
var q = undefined;
var order = 0; //descending
var limit = 2;
var json = 0;
var format = req.param("format");
var next = req.param("next");
var prev = req.param("prev");
streamsFindBy(format, undefined, undefined, 2, 10, next, prev, 0, function(err, rows) {
var starting_after = req.param("starting_after");
var ending_before = req.param("ending_before");
streamsFindBy(format, genre, q, order, limit, starting_after, ending_before, json, function(err, rows) {
if (err) {
res.send(503);
} else {
var error;
if (rows.length === 0) {
error = "No streams for this genre.";
error = "No streams for this format.";
} else {
error = false;
}
var next_url, prev_url;
if (rows.length == 10) {
var last_id = rows[9].id;
var next_url, prev_url, home_url;
var result = rows;
// if the full results don't appear show a start button
if((starting_after || ending_before) && result.length < limit) {
// clear pagination params
delete params.ending_before;
delete params.starting_after;
qstring = querystring.stringify(params);
home_url = req.path+'?'+qstring;
}
// on the fist page and any page with max results show the next button
if(!(starting_after || ending_before) || (result.length == limit)) {
var last_id = rows[result.length-1].id;
delete params.ending_before;
params.starting_after = last_id;
qstring = querystring.stringify(params);
next_url = req.path+'?'+qstring;
}
// any page with max results and any page with results with starting_after
// show the previous button
if((result.length == limit) || (result.length > 0 && starting_after)) {
var prev_id = rows[0].id;
next_url = req.path+'?next='+last_id;
prev_url = req.path+'?prev='+prev_id;
delete params.starting_after;
params.ending_before = prev_id;
qstring = querystring.stringify(params);
prev_url = req.path+'?'+qstring;
}
res.render("by_xx", {
title: format,
servers: rows,
error: error,
next: next_url,
prev: prev_url
prev: prev_url,
home: home_url
});
}
});
......
......@@ -8,11 +8,17 @@ function init(q, c, s) {
}
function byGenre(req, res) {
var params = req.params;
var format = undefined;
var q = undefined;
var order = 0; //descending
var limit = 2;
var json = 0;
var genre = req.param("genre");
var next = req.param("next");
var prev = req.param("prev");
streamsFindBy(undefined, genre, undefined, 2, 10, next, prev, 0, function(err, rows) {
console.log(req);
var starting_after = req.param("starting_after");
var ending_before = req.param("ending_before");
streamsFindBy(format, genre, q, order, limit, starting_after, ending_before, json, function(err, rows) {
if (err) {
res.send(503);
} else {
......@@ -22,19 +28,46 @@ function byGenre(req, res) {
} else {
error = false;
}
var next_url, prev_url;
if (rows.length == 10) {
var last_id = rows[9].id;
var next_url, prev_url, home_url;
var result = rows;
// if the full results don't appear show a start button
if((starting_after || ending_before) && result.length < limit) {
// clear pagination params
delete params.ending_before;
delete params.starting_after;
qstring = querystring.stringify(params);
home_url = req.path+'?'+qstring;
}
// on the fist page and any page with max results show the next button
if(!(starting_after || ending_before) || (result.length == limit)) {
var last_id = rows[result.length-1].id;
delete params.ending_before;
params.starting_after = last_id;
qstring = querystring.stringify(params);
next_url = req.path+'?'+qstring;
}
// any page with max results and any page with results with starting_after
// show the previous button
if((result.length == limit) || (result.length > 0 && starting_after)) {
var prev_id = rows[0].id;
next_url = req.path+'?next='+last_id;
prev_url = req.path+'?prev='+prev_id;
delete params.starting_after;
params.ending_before = prev_id;
qstring = querystring.stringify(params);
prev_url = req.path+'?'+qstring;
}
res.render("by_xx", {
title: genre,
servers: rows,
error: error,
next: next_url,
prev: prev_url
prev: prev_url,
home: home_url
});
}
});
......
......@@ -8,7 +8,15 @@ function init(q, c, s) {
}
function index(req, res) {
streamsFindBy(undefined, undefined, undefined, 0, 20, undefined, undefined, 0, function(err, rows) {
var format = undefined;
var genre = undefined;
var q = undefined;
var order = -1; //random
var limit = 10;
var starting_after = undefined;
var ending_before = undefined;
var json = 0;
streamsFindBy(format, genre, q, order, limit, starting_after, ending_before, json, function(err, rows) {
if (err) {
res.send(503);
} else {
......
......@@ -9,7 +9,6 @@ function init(q, c, x, s) {
}
function getListen(req, res) {
console.log(req.get('User-Agent'));
var userAgent = req.get('User-Agent');
filename = req.params.filename;
streamFindById(req.params.streamId, 0,function(err, rows) {
......
......@@ -8,90 +8,184 @@ function init(q, c) {
}
function getCachedStreams(format, genre, q, order, limit, next, prev, json, cb) {
var cacheString = JSON.stringify({"format":format, "genre":genre, "q":q, "order":order, "limit":limit, "next":next, "prev":prev, "json":json});
console.log(cacheString);
function getCachedStreams(format, genre, q, order, limit, starting_after, ending_before, json, cb) {
var cacheString = JSON.stringify({"format":format, "genre":genre, "q":q, "order":order, "limit":limit, "starting_after":starting_after, "ending_before":ending_before, "json":json});
cache.wrap(cacheString, function (_cb) {
var params = JSON.parse(cacheString);
findBy(params.format, params.genre, params.q, params.order, params.limit, params.next, params.prev, params.json, _cb);
findBy(params.format, params.genre, params.q, params.order, params.limit, params.starting_after, params.ending_before, params.json, _cb);
}, 5, cb);
}
/*
Takes in paramters {}
q = Search string
id, limit, offset, genre, format, order(0 Random, 1 Listeners Desc, 2 Listeners Asc)
limit, genre, format, order(0 Random, 1 Listeners Desc, 2 Listeners Asc)
starting_after, ending_before id for pagination
callback is called during the resulting query, should be
function(err, rows){}
err will have a message and might have a responsecode
Turns parameters into a sql query on streams
Example query built:
//convert to json
SELECT array_to_json(array_agg(row_to_json(t))) AS streams FROM (
SELECT s.id, s.stream_name, s.stream_type, s.description, s.songname, s.url, s.avg_listening_time, s.codec_sub_types, s.bitrate, s.hits, s.cm, s.samplerate, s.channels, s.quality, s.genres, array_agg(sm.listenurl) AS listenurls, SUM(sm.listeners) AS listeners, SUM(sm.max_listeners) AS max_listeners
FROM streams s
INNER JOIN server_mounts AS sm ON s.id = sm.stream_id
//where/and clauses for genres, codec_sub_types and q
WHERE $1 = ANY(s.genres)
AND $2 = ANY(s.codec_sub_types)
AND ($3 LIKE ANY(s.genres) OR s.stream_name LIKE $4 OR s.description LIKE $4 )
GROUP BY s.id
// having clause for starting at a starting_after or ending_before id
HAVING (sum(sm.listeners), s.id) > (
// sub query to get current listeners of the starting_after/ending_before id
(SELECT sum(sm.listeners) AS listeners
FROM streams s
INNER JOIN server_mounts AS sm ON s.id = sm.stream_id
WHERE s.id = $5
GROUP BY s.id
LIMIT 1)
,$5)
ORDER BY listeners ASC, s.id ASC
LIMIT $6
) t
*/
function findBy(format, genre, q, order, limit, next, prev, json, resultCallback)
function findBy(format, genre, q, order, limit, starting_after, ending_before, json, resultCallback)
{
console.log(format);
console.log(genre);
var queryStringJsonBeg = 'SELECT array_to_json(array_agg(row_to_json(t))) AS streams FROM (';
// error handling for arguments
// can't have both starting_after and ending_before
if(starting_after && ending_before) {
// call callback with error
resultCallback({"message":"can't have both starting_after and ending_before parameters","responsecode":400},[]);
return;
}
// need an order when doing pagination
if((starting_after || ending_before) && !(order==0 || order==1)) {
resultCallback({"message":"need an order parameter if using starting_after and ending_before parameters","responsecode":400},[]);
return;
}
// check to make sure params are not arrays can happen through
// ?param=value&param=value
var params = [format, genre, q, order, limit, starting_after, ending_before, json];
for(var i =0;i<params.length;i++)
{
if(Array.isArray(params[i])) {
resultCallback({"message":"cannot pass in arrays for parameters","responsecode":400},[]);
return;
}
}
// parse json if string
if (json instanceof String) {
json = parseInt(json);
}
// json can only be 0 or 1
if(json != 0 && json != 1) {
resultCallback({"message":"json must be 0 or 1","responsecode":400},[]);
return;
}
// check order is only -1, 0, or 1
if(order != undefined) {
if (order instanceof String) {
order = parseInt(order);
}
if( (order != -1 && order != 0 && order != 1)) {
resultCallback({"message":"order parameter must be -1, 1, or 2","responsecode":400},[]);
return;
}
}
var queryStringJsonBeg = 'SELECT array_to_json(array_agg(row_to_json(t))) AS streams FROM ( \n';
var queryStringJsonEnd = ' ) t';
var queryStringBeg = 'SELECT s.id, s.stream_name, s.stream_type, s.description \
,s.songname, s.url, s.avg_listening_time, s.codec_sub_types, s.bitrate, s.hits, \
s.cm, s.samplerate, s.channels, s.quality, s.genres, array_agg(sm.listenurl) AS listenurls, \
COUNT(sm.listeners) AS listeners, COUNT(sm.max_listeners) AS max_listeners \
FROM streams s \
INNER JOIN server_mounts AS sm ON s.id = sm.stream_id ';
var queryStringGroup = 'GROUP BY s.id ';
var queryStringBeg = 'SELECT s.id, s.stream_name, s.stream_type, s.description, ' +
's.songname, s.url, s.avg_listening_time, s.codec_sub_types, s.bitrate, s.hits, ' +
's.cm, s.samplerate, s.channels, s.quality, s.genres, array_agg(sm.listenurl) AS listenurls, ' +
'SUM(sm.listeners) AS listeners, SUM(sm.max_listeners) AS max_listeners ' +
'FROM streams s ' +
'INNER JOIN server_mounts AS sm ON s.id = sm.stream_id ';
var queryStringGroup = '\n GROUP BY s.id \n';
// final sql query string
var queryString;
// # for argument to be inserted into the sql string
var countItems = 1;
// list for arguments to be inserted into the sql string
var items = [];
// if json add the json conversion to the query
if(json) {
queryString = queryStringJsonBeg+queryStringBeg;
} else {
queryString = queryStringBeg;
}
var search;
// Add any WHERE/AND clauses necessary from the genre, format, q
if(genre) {
queryString += genreFormatModifier(countItems, 's.genres');
queryString += 'WHERE $'+countItems+' = ANY(s.genres) ';
countItems++;
items.push(genre);
}
if(format) {
queryString += genreFormatModifier(countItems, 's.codec_sub_types');
if(countItems == 1) {
queryString += 'WHERE $'+countItems+' = ANY(s.codec_sub_types) ';
} else {
queryString += 'AND $'+countItems+' = ANY(s.codec_sub_types) ';
}
countItems++;
items.push(format);
}
if(q) {
search = q;
var add = false;
var doSearch = false;
var searchWord = '';
var searchArray ='(';
if(Array.isArray(search)) {
search = search[0];
}
search = search.split(' ');
var search = q.split(' ');
var searchString = 'WHERE (';
if(countItems > 1) {
searchString = 'AND (';
}
for(var i=0;i<search.length;i++)
{
// limit search words to 3
if(i>3) {
break;
}
// minimum word length of 3
if(search[i].length<3) {
continue;
}
add = true;
doSearch = true;
//maximum word length of 10
if(search[i].length>10) {
search[i] = search[i].substring(0,10);
}
// add match anything to the end of the string
searchWord += search[i]+'%';
searchString += '$'+countItems+' LIKE ANY(s.genres) OR ';
countItems++;
items.push(search[i]);
}
if(add) {
if(doSearch) {
searchString += 's.stream_name LIKE $'+countItems+' OR s.description LIKE $'+countItems+' ';
items.push(searchWord);
countItems++;
......@@ -100,73 +194,96 @@ function findBy(format, genre, q, order, limit, next, prev, json, resultCallback
queryString += searchString;
} else {
// search query won't return any results
resultCallback(0,[],{});
resultCallback({"message":"search string not long enough"},[]);
return;
}
}
if(next && (order ==1 || order ==2)) {
searchString = 'WHERE ';
if(countItems > 1) {
searchString = 'AND ';
}
var sign;
// add in the Group BY
queryString = queryString + queryStringGroup;
//subQuery to get # listeners for the starting_after/ending_before id
var subQuery = '(SELECT sum(sm.listeners) AS listeners FROM streams s ' +
'INNER JOIN server_mounts AS sm ON s.id = sm.stream_id ' +
'WHERE s.id = $'+ countItems +
'GROUP BY s.id ' +
'LIMIT 1)';
// pagination need to use HAVING because sum(sm.listeners) will return the
// correct value
if(ending_before) {
if( order==1 ){
sign = '<';
order = 2;
queryString += 'HAVING (sum(sm.listeners), s.id) < ' + '( ' + subQuery + ' ,$' + countItems + ') ';
} else {
sign = '>';
order = 1;
queryString += 'HAVING (sum(sm.listeners), s.id) > ' + '(' + subQuery + ' ,$' + countItems + ') ';
}
queryString += searchString + 's.id ' + sign + ' ' + '$' + countItems + ' ';
items.push(next);
items.push(ending_before);
countItems++;
}
else if(prev && (order ==1 || order ==2)) {
searchString = 'WHERE ';
if(countItems > 1) {
searchString = 'AND ';
}
var sign;
if( order==1 ){
sign = '>';
else if(starting_after) {
if( order==0 ){
queryString += 'HAVING (sum(sm.listeners), s.id) < ' + '(' + subQuery + ' ,$' + countItems + ') ';
} else {
sign = '<';
queryString += 'HAVING (sum(sm.listeners), s.id) > ' + '(' + subQuery + ' ,$' + countItems + ') ';