new database schema, json api and website updated

README

+TODO:
+- Add the rest of the url checks to yp-cgi.js, also might turn the checks into an
+admin interface
+- Add meaningful error codes to the json api, not just returning empty []
+- Add more options to the api if needed (Grouping of streams by Channel, geographical area)
+    https://trac.xiph.org/ticket/1958
+- Create documenation for the json api
+- Finish the html directory site
+- Add caching to the json api ? maybe

app.js

@@ -5,17 +5,24 @@ var express         = require('express'),
     cache_manager   = require('cache-manager'),
     bodyParser      = require('body-parser'),
     qs              = require('qs'),
-    conf            = require('konphyg')(__dirname + '/config');
+    conf            = require('konphyg')(__dirname + '/config'),
+    validator       = require('validator');
 
 var cache           = cache_manager.caching({store: "memory", max: 100, ttl: 10});
 var app             = express();
 var config          = conf('config');
 
+
 /* Controllers */
-var index           = require('./controllers/index.js')(query, cache);
-var genres          = require('./controllers/genres.js')(query, cache);
-var formats         = require('./controllers/formats.js')(query, cache);
-var yp_cgi          = require('./controllers/yp-cgi.js')(query, qs);
+var streamApi       = require('./controllers/stream-api.js')(query, cache);
+var index           = require('./controllers/index.js')(query, cache, streamApi);
+var genres          = require('./controllers/genres.js')(query, cache, streamApi);
+var formats         = require('./controllers/formats.js')(query, cache, streamApi);
+var yp_cgi          = require('./controllers/yp-cgi.js')(query, qs, validator);
+var listen          = require('./controllers/listen.js')(query, qs, streamApi)
+
+
+
 
 query.connectionParameters = config.db;
 
@@ -31,11 +38,60 @@ app.engine('html', swig.renderFile);
 app.set('view engine', 'html');
 app.set('views', __dirname + '/views');
 
-/* Routes */
+/* Website Routes */
 app.get('/', index);
+app.post('/cgi-bin/yp-cgi', yp_cgi);
 app.get('/by_genre/:genre', genres);
 app.get('/by_format/:format', formats);
-app.post('/cgi-bin/yp-cgi', yp_cgi);
+app.get('/listen/:streamId/:filename',listen)
+
+/* JSON API */
+function respond(res, err, rows, result)
+{
+    if(err)
+    {
+        res.send([]);
+    }
+    else
+    {
+        res.send(rows);
+    }
+}
+app.get('/streams/', function(req,res){
+    res.set('Content-Type', 'application/json')
+    streamApi(req.query,function(err, rows,result){
+        respond(res,err,rows,result)
+    });
+});
+app.get('/streams/:streamId',function(req,res){
+    res.set('Content-Type', 'application/json')
+    params = {'id':req.params.streamId}
+    streamApi(params,function(err,rows,result){
+        if(err || result.rowCount != 1)
+        {
+            res.send([]);
+        }
+        else
+        {
+            res.send(rows[0]);
+        }
+    });
+})
+app.get('/genres/', function(req, res) {
+    res.set('Content-Type', 'application/json')
+    var genresq = 'SELECT DISTINCT val FROM (SELECT unnest(genres) as val FROM streams) s;';
+    query(genresq, function(err, rows, result) {
+        respond(res,err,rows,result)
+    });
+});
+app.get('/formats/', function(req, res) {
+    res.set('Content-Type', 'application/json')
+    var formats = 'SELECT DISTINCT val FROM (SELECT unnest(codec_sub_types) as val FROM streams) s;';
+    query(formats, function(err, rows, result) {
+        respond(res,err,rows,result)
+    });
+});
+
 
 var server = app.listen(3000, function() {
     console.log('Listening on port %d', server.address().port);
@@ -46,78 +102,9 @@ process.on('SIGINT', function() {
     process.exit();
 });
 
+/*
 var interval = setInterval(deleteOldServers, 240000, 4);
 function deleteOldServers(time, cb) {
     query('DELETE FROM servers WHERE lasttouch < NOW() - INTERVAL \'' + time.toString() + ' minutes\';', cb);
 }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-app.get('/view/', function(req, res) {
-    //console.log("[GET " + req.originalUrl + "]:")
-    res.set('Content-Type', 'application/json')
-    if(!req.param('genre')) {
-        query('SELECT array_to_json(array_agg(row_to_json(servers))) FROM servers;', function(err, rows, result) {
-            //console.log(err)
-            res.send(rows[0].array_to_json)
-        });
-    } else {
-        query('SELECT array_to_json(array_agg(row_to_json(servers))) FROM servers WHERE $1 = ANY (genres);',
-            [req.param('genre')],
-            function(err, rows, result) {
-            //console.log(err)
-            res.send(rows[0].array_to_json)
-        });
-    }
-});
-
-app.get('/insert/', function(req, res) {
-    //console.log("[GET " + req.originalUrl + "]:")
-    query('INSERT INTO servers VALUES (\
-        uuid_generate_v4(),\
-        $1,\
-        $2,\
-        $3,\
-        $4,\
-        $5,\
-        $6,\
-        $7,\
-        $8,\
-        $9,\
-        now()\
-        );', [req.param('sn'), req.param('type'), [req.param('genres')], req.param('b'), req.param('listenurl'),
-    req.param('cpswd'), req.param('desc'), req.param('url'), [req.param('stype')]], function(err, rows, result) {
-        //console.log(err)
-        res.send(err)
-    });
-});
-
-app.get('/genres/', function(req, res) {
-  var genresq = 'SELECT DISTINCT val FROM (SELECT unnest(genres) as val FROM servers) s;';
-  query(genresq, function(err, rows, result) {
-    res.send(rows);
-  });
-});
\ No newline at end of file
+*/

config/config.json

 {
-	"db" : "postgres://user:passwd@host:port/dbname"
+	"db" : "postgres://marcof@localhost/marcof"
 }
\ No newline at end of file

controllers/formats.js

-var query, cache;
+var query, cache, streamApi;
 
-function init(q, c) {
+function init(q, c, s) {
     query = q;
     cache = c;
+    streamApi = s;
     return byFormat;
 }
 
@@ -33,11 +34,10 @@ function getCachedFormatStreams(format, cb) {
 }
 
 function getFormatStreams(format, cb) {
-    query("SELECT id, server_name, server_type, genres, bitrate, listenurl, description, url, \
-           codec_sub_types, songname, listeners FROM servers WHERE $1 = ANY (codec_sub_types);", [format],
-    function(err, rows, result) {
+    params = {'format':format}
+    streamApi(params,function(err, rows, result) {
         cb(err, result);
     });
 }
 
-module.exports = init;
\ No newline at end of file
+module.exports = init;

controllers/genres.js

-var query, cache;
+var query, cache, streamApi;
 
-function init(q, c) {
+function init(q, c, s) {
     query = q;
     cache = c;
+    streamApi = s;
     return byGenre;
 }
 
@@ -33,11 +34,10 @@ function getCachedGenreStreams(genre, cb) {
 }
 
 function getGenreStreams(genre, cb) {
-    query("SELECT id, server_name, server_type, genres, bitrate, listenurl, description, url, \
-           codec_sub_types, songname, listeners FROM servers WHERE $1 = ANY (genres);", [genre],
-    function(err, rows, result) {
+    params = {'genre':genre}
+    streamApi(params,function(err, rows, result) {
         cb(err, result);
     });
 }
 
-module.exports = init;
\ No newline at end of file
+module.exports = init;

controllers/index.js

-var query, cache;
+var query, cache, streamApi;
 
-function init(q, c) {
+function init(q, c, s) {
     query = q;
     cache = c;
+    streamApi = s;
     return index;
 }
 
@@ -17,6 +18,7 @@ function index(req, res) {
             } else {
                 error = false;
             }
+            console.log(result.rows)
             res.render("index", {
                 title: '',
                 servers: result.rows,
@@ -33,11 +35,10 @@ function getCachedRandomStreams(count, cb) {
 }
 
 function getRandomStreams(count, cb) {
-    query('SELECT id, server_name, server_type, genres, bitrate, listenurl, description, url, \
-           codec_sub_types, songname, listeners FROM servers ORDER BY random() LIMIT $1;', [count],
-    function(err, rows, result) {
+    params = {'limit':count,'order':0}
+    streamApi(params,function(err, rows, result) {
         cb(err, result);
     });
 }
 
-module.exports = init;
\ No newline at end of file
+module.exports = init;

controllers/listen.js

+var query, cache, streamApi;
+
+function init(q, c, s) {
+    query = q;
+    cache = c;
+    streamApi = s;
+    return getListen;
+}
+
+function getListen(req, res) {
+    params = {'id':req.params.streamId}
+    console.log(req.get('User-Agent'))
+    var userAgent = req.get('User-Agent')
+    filename = req.params.filename
+    streamApi(params,function(err, rows, result) {
+        if(err || result.rowCount != 1)
+        {
+            res.status(404);
+            res.send('Stream not in the Database');
+            return
+        }
+        var extension = filename.substr(filename.lastIndexOf('.')+1)
+        console.log(extension)
+        if(extension != 'm3u' && extension != 'xspf')
+        {
+            res.status(400);
+            res.send('Not the right extension');
+            return
+        }
+        var listenurls = rows[0].listenurls
+        if(extension == 'm3u')
+        {
+            var outputString = ''
+            for (var i = 0; i < listenurls.length; i++)
+            {
+                outputString += listenurls[i]+'\r\n'
+            }
+            res.set('Content-Type','audio/x-mpegurl')
+            if(userAgent.indexOf('/MSIE 5.5/') != -1)
+            {
+                res.set('Content-Disposition','inline; filename="listen.m3u"')
+            }
+            else
+            {
+                res.set('Content-Disposition','filename="listen.m3u"')
+            }
+            res.send(outputString)
+        }
+        else if(extension == 'xspf')
+        {
+            var outputString = '<?xml version="1.0" encoding="UTF-8"?>\
+	        \r\n<playlist version="1" xmlns="http://xspf.org/ns/0/">\
+	        \r\n    <title>'+rows[0].stream_name+'</title>\
+	        \r\n    <info>'+rows[0].url+'</info>\
+	        \r\n    <trackList>'
+            for (var i = 0; i < listenurls.length; i++)
+            {
+                outputString +='\r\n        <track><location>'+listenurls[i]+'</location></track>'
+            }
+            outputString +='\r\n    </trackList>\r\n</playlist>'
+            res.set('Content-Type','application/xspf+xml')
+            res.set('Content-Disposition','filename="listen.xspf"')
+            if(userAgent.indexOf('/MSIE 5.5/') != -1)
+            {
+                res.set('Content-Disposition','inline; filename="listen.xspf"')
+            }
+            else
+            {
+                res.set('Content-Disposition','filename="listen.xspf"')
+            }
+            res.send(outputString)
+        }
+    });
+}
+
+module.exports = init;

controllers/stream-api.js

+var query, cache;
+
+function init(q, c) {
+    query = q;
+    cache = c;
+    return streamApi;
+}
+
+
+/*
+    Takes in paramters {}
+    q = Search string
+    id, limit, offset, genre, format, order(0 Random, 1 Listeners Desc, 2 Listeners Asc)
+    callback is called during the resulting query, should be
+    function(err, rows, result){}
+
+    Turns parameters into a sql query on streams
+*/
+function streamApi(params,callback)
+{
+    var allStreamsBeg = 'SELECT s.id, s.stream_name, s.stream_type, s.description,s.songname, s.url, s.avg_listening_time, s.codec_sub_types, s.bitrate, s.hits, s.cm, s.samplerate, s.channels, s.quality, s.genres, array_agg(sm.listenurl) AS listenurls, COUNT(sm.listeners) AS listeners, COUNT(sm.max_listeners) AS max_listeners FROM streams s INNER JOIN server_mounts AS sm ON s.id = sm.stream_id '
+    var allStreamsGroup = 'GROUP BY s.id '
+    var countItems = 1
+    var items = []
+    var queryString = allStreamsBeg
+    var genre, format, streamId, search, order, limit, offset;
+    if(params.genre)
+    {
+        genre = params.genre
+        if(Array.isArray(genre))
+        {
+            for(var i =0; i <genre.length;i++)
+            {
+                queryString += genreFormatModifier(countItems, 's.genres')
+                countItems++
+                items.push(genre[i])
+            }
+        }
+        else
+        {
+          queryString += genreFormatModifier(countItems, 's.genres')
+          countItems++
+          items.push(genre)
+        }
+    }
+    if(params.format)
+    {
+        format = params.format
+        if(Array.isArray(format))
+        {
+            for(var i =0; i <genre.length;i++)
+            {
+                queryString += genreFormatModifier(countItems, 's.codec_sub_types')
+                countItems++
+                items.push(format[i])
+            }
+        }
+        else
+        {
+          queryString += genreFormatModifier(countItems, 's.codec_sub_types')
+          countItems++
+          items.push(format)
+        }
+    }
+    if(params.id)
+    {
+        streamId = params.id
+        if(countItems > 1)
+        {
+            queryString += 'AND $'+countItems+' = s.id '
+        }
+        else
+        {
+            queryString += 'WHERE $'+countItems+' = s.id '
+        }
+        countItems++
+        items.push(streamId)
+    }
+    if(params.q)
+    {
+        search = params.q
+        var add = false;
+        var searchWord = '';
+        var searchArray ='(';
+        if(Array.isArray(search))
+        {
+            search = search[0]
+        }
+        search = search.split(' ')
+        var searchString = 'WHERE ('
+        if(countItems > 1)
+        {
+            searchString = 'AND ('
+        }
+        for(var i=0;i<search.length;i++)
+        {
+            if(i>3)
+            {
+                break;
+            }
+            // minimum word length of 3
+            if(search[i].length<3)
+            {
+                continue;
+            }
+            add = true
+            //maximum word length of 10
+            if(search[i].length>10)
+            {
+                search[i] = search[i].substring(0,10)
+            }
+            searchWord += search[i]+'%'
+            searchString += '$'+countItems+' LIKE ANY(s.genres) OR '
+            countItems++
+            items.push(search[i])
+        }
+        if(add)
+        {
+            searchString += 's.stream_name LIKE $'+countItems
+            +' OR s.description LIKE $'+countItems+' '
+            items.push(searchWord)
+            countItems++
+
+            searchString += ') '
+            queryString += searchString
+        }
+        else
+        {
+            // search query won't return any results
+            callback(0,[],{})
+            return
+        }
+    }
+    // add in the Group BY
+    queryString = queryString + allStreamsGroup
+    if(params.order)
+    {
+        order = params.order
+        if(order == 0)
+        {
+            queryString += 'ORDER BY random() '
+        }
+        else if(order == 1)
+        {
+            queryString += 'ORDER BY listeners DESC'
+        }
+        else if(order == 2)
+        {
+            queryString += 'ORDER BY listeners ASC'
+        }
+
+    }
+    if(params.limit) {
+        limit = params.limit
+        if(Array.isArray(limit))
+        {
+            limit = limit[0]
+        }
+        else
+        {
+          queryString += 'LIMIT $'+countItems+' '
+          countItems++
+          items.push(limit)
+        }
+    }
+    if(params.offset) {
+        offset = params.offset
+        if(Array.isArray(offset))
+        {
+            offset = offset[0]
+        }
+        else
+        {
+          queryString += 'OFFSET $'+countItems+' '
+          countItems++
+          items.push(offset)
+        }
+    }
+    //console.log(queryString)
+    query(queryString,items,callback)
+}
+function genreFormatModifier(count, type)
+{
+    if(count > 1)
+    {
+        return ('AND $'+count+' = ANY('+type+') ')
+    }
+    else
+    {
+        return ('WHERE $'+count+' = ANY('+type+') ')
+    }
+}
+
+module.exports = init;

controllers/yp-cgi.js

-var query, qs;
+var query, qs, validator;
+var async = require('async');
 
-function init(q, q_) {
+function init(q, q_, v) {
     query = q;
     qs = q_;
+    validator = v;
     return dispatcher;
 }
 
@@ -10,72 +12,214 @@ function dispatcher(req, res) {
     if(req.body.action == "add") {
         ypAdd(req, res);
     } else if(req.body.action == "touch") {
-        ypTouch(req, res);   
+        ypTouch(req, res);
     } else if (req.body.action == "remove") {
         ypRemove(req, res);
     }
 }
 
+function checkPresent(toCheck, check)
+{
+    for(var i=0;i<check.length;i++)
+    {
+        if(!(check[i] in toCheck))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+function multiIndexOf(toCheck, check)
+{
+  for(var i=0;i<check.length;i++)
+  {
+    if(toCheck.indexOf(check[i]) != -1)
+    {
+        return false;
+    }
+  }
+  return true;
+}
+
 function ypAdd(req, res) {
-    if (req.body.listenurl.indexOf("http://localhost") != -1 ||
-        req.body.listenurl.indexOf("https://localhost") != -1) {
-        ypRes(res, false, "Your hostname is localhost, this is an indicator of a misconfigured server", -1, null);
+    var start = new Date().getTime();
+    var params = req.body;
+    // check mandatory arguments
+    var mandatoryArgs = ['sn', 'type', 'genre', 'listenurl']
+    var illegalListenUrls = ['dev.local','testvm.hivane.net',
+                            'backup.abidingradio.com']
+    var abuseIps = ['92.246.30.112']
+
+    var misconfiguredUrls = ['hostingcenter.com']
+    var misconfiguredIps = ['192.240.97.','192.240.102.','50.7..']
+
+    //need to add others
+    var defaultServerNames = ['Unspecified name','This is my server name',
+                             'Stream Name','My Station name']
+
+    if( checkPresent(params, mandatoryArgs) == false)
+    {
+        ypRes(res, false, "Not enough arguments", -1, null);
+        return;
+    }
+
+    if( validator.isURL(params.listenurl) == false)
+    {
+        ypRes(res, false, "Not a real listenurl", -1, null);
+        return;
+    }
+
+    if( multiIndexOf(params.listenurl, illegalListenUrls) == false)
+    {
+        ypRes(res, false, "Illegal listen_url. Don't test against a production \
+        server, thanks! ", -1, null);
         return;
     }
-    if (req.body.sn === 'Unspecified name') {
-        ypRes(res, false, "You have to specify a name for your stream.", -1, null);
+
+    if( multiIndexOf(params.listenurl, abuseIps) == false)
+    {
+        ypRes(res, false, "Your server has been banned for abuse, have a nice \
+        day!", -1, null);
         return;
     }
 
+    if( multiIndexOf(params.listenurl, misconfiguredUrls) == false)// || multiIndexOf(ip,misconfiguredIps))
+    {
+        ypRes(res, false, "The network range in which your server resides has \
+        been suspended due to a high number of wrongly configured servers! Please\
+         contact webmaster@xiph.org urgently!", -1, null);
+        return;
+    }
+
+    if( multiIndexOf(params.sn, defaultServerNames) == false)
+    {
+        ypRes(res, false, "Default stream name detected, please configure your \
+        source client, thanks! ", -1, null);
+        return;
+    }
+
+    //parse the body
     var final = parseBody(req.body);
-    var insert = 'INSERT INTO servers (id, lasttouch, server_name, server_type, genres, bitrate, listenurl, \
-                    cluster_pass, description, url, codec_sub_types, channels, samplerate, quality) \
-                    VALUES (uuid_generate_v4(), now(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \
-                    RETURNING id;'
-    query(insert, [final.server_name, final.server_type, final.genres, final.bitrate,
-                    final.listenurl, final.cluster_pass, final.description, final.url,
-                    final.codec_sub_types, final.channels, final.samplerate, final.quality],
-    function(err, rows, result) {
-        console.log(err);
-        if (err) {
-            ypRes(res, false, "Error adding the stream", -1, null);
-        } else {
-            ypRes(res, true, "Successfully Added", rows[0].id, 200);
-        };
+
+    var insertStream = 'INSERT INTO streams (stream_name, stream_type, genres, bitrate, cluster_pass, description, url, codec_sub_types, channels, samplerate, quality) \
+                        VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) \
+                        RETURNING id;'
+    var insertServerMount = 'INSERT INTO server_mounts (sid, stream_id, lasttouch, listenurl) \
+                    VALUES (uuid_generate_v4(), $1, now(), $2) \