Commit e3486aa5 authored by Marco Flowers's avatar Marco Flowers

Added testing, created client emulator, updated yp

Added basic tests for adding streams, and created an emulator of clients
adding/touching/deleting streams. Updated ypTouch and application types
allowed.
parent 7900f62c
......@@ -9,20 +9,37 @@ var express = require('express'),
validator = require('validator'),
xmlbuilder = require('xmlbuilder');
querystring = require('querystring');
bunyan = require('bunyan');
var cache = cache_manager.caching({store: "memory", max: 100, ttl: 10});
var app = express();
var config = conf.all().config;
// if logging is needed
var log = bunyan.createLogger({
name: 'dev',
streams: [
{
level: 'info',
stream: process.stdout // log INFO and above to stdout
},
{
level: 'error',
path: __dirname + '/error.log' // log ERROR and above to a file
}
]
});
/* Controllers */
var stats = require('./controllers/stats.js')(query, cache);
var streamsFindBy = require('./controllers/stream-api.js')(query, cache);
var streamFindById = require('./controllers/stream-by-id.js')(query, cache);
var index = require('./controllers/index.js')(query, cache, streamsFindBy, stats);
var yp_cgi = require('./controllers/yp-cgi.js')(query, qs, validator, config);
var listen = require('./controllers/listen.js')(query, qs, xmlbuilder, streamFindById);
var search = require('./controllers/search.js')(query, cache, streamsFindBy, stats);
var stats = require('./controllers/stats.js')(query, cache, log);
var streamsFindBy = require('./controllers/stream-api.js')(query, cache, log);
var streamFindById = require('./controllers/stream-by-id.js')(query, cache, log);
var index = require('./controllers/index.js')(query, cache, streamsFindBy, stats, log);
var yp_cgi = require('./controllers/yp-cgi.js')(query, qs, validator, config, log);
var listen = require('./controllers/listen.js')(query, qs, xmlbuilder, streamFindById, log);
var search = require('./controllers/search.js')(query, cache, streamsFindBy, stats, log);
/*
To rerun api docs after modifying the apidoc comments use the command
......@@ -37,7 +54,6 @@ swig.setDefaults({ cache: (process.env.NODE_ENV === 'production') ? true : false
app.use(morgan((process.env.NODE_ENV === 'production') ? 'short' : 'dev'));
app.use(bodyParser.urlencoded({ extended: true }));
app.use('/assets', express.static(__dirname + '/assets'));
app.engine('html', swig.renderFile);
app.set('view engine', 'html');
......@@ -271,3 +287,5 @@ function deleteOldServers(time, cb) {
query('DELETE FROM servers WHERE lasttouch < NOW() - INTERVAL \'' + time.toString() + ' minutes\';', cb);
}
*/
module.exports.getApp = app;
var query, cache, streamsFindBy, stats;
var query, cache, streamsFindBy, stats, log;
function init(q, c, s, st) {
function init(q, c, s, st, l) {
query = q;
cache = c;
streamsFindBy = s;
stats = st;
log = l;
return index;
}
......
var query, cache, xmlbuilder, streamFindById;
var query, cache, xmlbuilder, streamFindById, log;
function init(q, c, x, s) {
function init(q, c, x, s, l) {
query = q;
cache = c;
streamFindById = s;
xmlbuilder = x;
log = l;
return getListen;
}
......
var query, cache, streamsFindBy, stats;
var query, cache, streamsFindBy, stats, log;
function init(q, c, s, st) {
function init(q, c, s, st, l) {
query = q;
cache = c;
streamsFindBy = s;
stats = st;
log = l;
return bySearch;
}
......@@ -14,7 +15,7 @@ function bySearch(req, res) {
var genre = req.param("genre");
var q = req.param("q");
var order = 0; //descending
var limit = 2;
var limit = 10;
var json = 0;
var starting_after = req.param("starting_after");
var ending_before = req.param("ending_before");
......@@ -24,49 +25,53 @@ function bySearch(req, res) {
res.send(503);
} else {
var error;
if (rows.length === 0) {
error = false;
if (rows.length == 0) {
error = "No streams for this search.";
} else {
error = false;
if(starting_after) {
error = "No more streams for this search";
}
}
var next_url, prev_url, home_url;
var result = rows;
var next_url, prev_url, home_url;
var result = rows;
// if the full results don't appear show a start button
if((starting_after || ending_before) && result.length < limit) {
// clear pagination params
delete params.ending_before;
delete params.starting_after;
delete params.last_count;
qstring = querystring.stringify(params);
home_url = req.path+'?'+qstring;
}
// if the full results don't appear show a start button
if((starting_after || ending_before) && result.length < limit) {
// clear pagination params
delete params.ending_before;
delete params.starting_after;
delete params.last_count;
qstring = querystring.stringify(params);
home_url = req.path+'?'+qstring;
}
// on the fist page and any page with max results show the next button
if(!(starting_after || ending_before) || (result.length == limit)) {
var last_id = rows[result.length-1].id;
var last_count = rows[result.length-1].listeners;
delete params.ending_before;
params.starting_after = last_id;
params.last_listener_count = last_count;
qstring = querystring.stringify(params);
next_url = req.path+'?'+qstring;
}
// any page with max results show the next button
if((result.length == limit)) {
var last_id = rows[result.length-1].id;
var last_count = rows[result.length-1].listeners ;
console.log(last_count)
delete params.ending_before;
params.starting_after = last_id;
params.last_listener_count = last_count;
qstring = querystring.stringify(params);
next_url = req.path+'?'+qstring;
}
// any page with max results and any page with results with starting_after
// show the previous button
if((result.length == limit) || (result.length > 0 && starting_after)) {
var prev_id = rows[0].id;
var prev_count = rows[0].listeners;
delete params.starting_after;
params.ending_before = prev_id;
params.last_listener_count = prev_count;
// any page with max results and any page with results with starting_after
// show the previous button
if((result.length == limit) || (result.length > 0 && starting_after)) {
var prev_id = rows[0].id;
var prev_count = rows[0].listeners;
delete params.starting_after;
params.ending_before = prev_id;
params.last_listener_count = prev_count;
qstring = querystring.stringify(params);
prev_url = req.path+'?'+qstring;
}
qstring = querystring.stringify(params);
prev_url = req.path+'?'+qstring;
}
stats(function(errorStats, stats) {
res.render("by_xx", {
title: "Search",
......
var query, cache;
var query, cache, log;
var async = require('async');
function init(q, c) {
function init(q, c, l) {
query = q;
cache = c;
log;
return getCachedStats;
}
......@@ -18,13 +19,14 @@ function getCachedStats(cb) {
function getStats(resultCallback)
{
console.log("go")
var genresq = 'SELECT DISTINCT val FROM (SELECT unnest(genres) as val FROM streams) s;';
var formats = 'SELECT DISTINCT val FROM (SELECT unnest(codec_sub_types) as val FROM streams) s;';
var formatsq = 'SELECT codec, COUNT(*) as count FROM (SELECT unnest(s.codec_sub_types) AS codec' +
' FROM streams s) t GROUP BY codec ORDER BY count desc'
var baseStatsQuery ='SELECT COUNT(*) AS total FROM streams AS s '+
var genresq = 'SELECT genre, COUNT(*) as count FROM (SELECT unnest(s.genres) AS genre' +
' FROM streams s) t GROUP BY genre ORDER BY count desc'
var totalStreams ='SELECT COUNT(*) AS total FROM streams AS s '+
'INNER JOIN server_mounts AS sm ON s.id = sm.stream_id ';
var wheremodifier = 'WHERE $1 = ANY(s.codec_sub_types)'
var stats = {}
stats.statistics = {}
......@@ -34,49 +36,19 @@ function getStats(resultCallback)
},
function(rows, result, cb) {
stats["genres"] = rows
query(formats, cb);
console.log(rows)
query(formatsq, cb);
},
function(rows, result, cb) {
stats["formats"] = rows
query(baseStatsQuery, cb)
query(totalStreams, cb)
},
function(rows, result, cb) {
stats.statistics["Total Streams"] = rows[0].total
query(baseStatsQuery+wheremodifier,['Vorbis'], cb)
},
function(rows, result, cb) {
stats.statistics["Ogg Vorbis"] = rows[0].total
query(baseStatsQuery+wheremodifier,['Opus'], cb)
},
function(rows, result, cb) {
stats.statistics["Opus"] = rows[0].total
query(baseStatsQuery+wheremodifier,['Theora'], cb)
},
function(rows, result, cb) {
stats.statistics["Theora"] = rows[0].total
query(baseStatsQuery+wheremodifier,['MP3'], cb)
},
function(rows, result, cb) {
stats.statistics["MP3"] = rows[0].total
query(baseStatsQuery+wheremodifier,['WebM'], cb)
},
function(rows, result, cb) {
stats.statistics["Webm"] = rows[0].total
query(baseStatsQuery+wheremodifier,['AAC'], cb)
},
function(rows, result, cb) {
stats.statistics["AAC"] = rows[0].total
query(baseStatsQuery+wheremodifier,['AAC+'], cb)
},
function(rows, result, cb) {
stats.statistics["AAC+"] = rows[0].total
query(baseStatsQuery+wheremodifier,['NSV'], cb)
resultCallback(null, stats);
},
function(rows, result, cb) {
stats.statistics["NSV"] = rows[0].total
resultCallback(null, stats)
}
],function (err, result) {
console.log("err")
if(err) {
resultCallback(err, null)
}
......
var query, cache;
var query, cache, log;
function init(q, c) {
function init(q, c, l) {
query = q;
cache = c;
log = l;
return getCachedStreams;
}
......@@ -11,7 +12,6 @@ function init(q, c) {
function getCachedStreams(format, genre, q, order, limit, starting_after, ending_before,last_listener_count, json, cb) {
var cacheString = JSON.stringify({"format":format, "genre":genre, "q":q, "order":order, "limit":limit, "starting_after":starting_after, "ending_before":ending_before, "last_listener_count":last_listener_count, "json":json});
console.log(cacheString);
cache.wrap(cacheString, function (_cb) {
var params = JSON.parse(cacheString);
findBy(params.format, params.genre, params.q, params.order, params.limit, params.starting_after, params.ending_before, params.last_listener_count, params.json, _cb);
......
var query, cache;
var query, cache, log;
function init(q, c) {
function init(q, c, l) {
query = q;
cache = c;
log = l;
return getCachedStreamById;
}
......@@ -42,7 +43,6 @@ function findById(id, json, resultCallback)
if(json) {
queryString += queryStringJsonEnd;
}
console.log(queryString);
query(queryString, items, resultCallback);
}
......
var query, qs, validator, config;
var query, qs, validator, config, log;
var async = require('async');
var foreign_key_violation = '23503';
var duplicate_key_violation = '23505';
function init(q, q_, v, c) {
function init(q, q_, v, c, l) {
query = q;
qs = q_;
validator = v;
config = c;
log = l;
return dispatcher;
}
......@@ -18,6 +20,10 @@ function dispatcher(req, res) {
} else if (req.body.action == "remove") {
ypRemove(req, res);
}
else {
ypRes(res, false, "Need action argument", -1, null);
}
return;
}
function checkPresent(toCheck, check)
......@@ -113,8 +119,11 @@ function ypAdd(req, res) {
],
function(err, result) {
if(err) {
console.log(err);
ypRes(res, false, "Server error", -1, null);
if(err.code == duplicate_key_violation) {
ypRes(res, false, 'Entry already in the YP. If this happens constantly your server is misconfigured! Verify that "listenurl" is reachable!', -1, null);
} else {
ypRes(res, false, "Server error", -1, null);
}
}
});
}
......@@ -123,26 +132,35 @@ function ypAdd(req, res) {
function ypTouch(req, res) {
var final = parseBody(req.body);
var params;
var touch = 'UPDATE server_mounts SET lasttouch = now(), songname = $2, \
listeners = $3, max_listeners = $4 WHERE sid = $1 Returning stream_id;';
var updateStream = 'UPDATE streams SET songname = $1, codec_sub_types = $2 \
WHERE id = $3;';
var updateServerMount = 'UPDATE server_mounts SET lasttouch = now(), songname = $2, listeners = $3'+
', max_listeners = $4 WHERE sid = $1 Returning stream_id;';
var updateStream = 'UPDATE streams SET songname = $2, codec_sub_types = $3'+
'WHERE id = $1;';
async.waterfall([
function start(cb) {
// update the server_mount
query(touch, [final.id, final.songname, final.listeners, final.max_listeners], cb);
if(final.id == undefined) {
ypRes(res, false, "Not enough arguments", -1, null);
return;
}
if(final.listeners === undefined) {
final.listeners = 0;
}
if(final.max_listeners === undefined) {
final.max_listeners = 0;
}
query(updateServerMount, [final.id, final.songname, final.listeners, final.max_listeners], cb);
},
function(row,result, cb) {
if(result.rowCount != 1) {
// end with error
cb(1);
return;
}
//update the stream song
params = [final.songname,final.codec_sub_types,row[0].stream_id];
params = [row[0].stream_id, final.songname,final.codec_sub_types];
if(!final.codec_sub_types) {
updateStream = 'UPDATE streams SET songname = $1 \
WHERE id = $3;';
params = [final.songname,row[0].stream_id];
updateStream = 'UPDATE streams SET songname = $2 \
WHERE id = $1;';
params = [row[0].stream_id, final.songname];
}
query(updateStream, params, cb);
},
......@@ -169,6 +187,7 @@ function ypRemove(req, res) {
function(row,result, cb) {
if(result.rowCount != 1) {
cb(1);
return;
}
var idRow = row[0].stream_id;
query(removeStream, [idRow], cb);
......@@ -176,7 +195,6 @@ function ypRemove(req, res) {
function(row,result, cb) {
// if no error on delete(foreign key constraint then succesfully removed)
ypRes(res, true, "Successfully removed", null, null);
console.log("Successfuly removed");
},
],
function(err, result) {
......@@ -308,13 +326,13 @@ function parseBody(body) {
function generateSubType(type) {
var stype;
if (type === 'audio/opus') {
if (type === 'audio/opus' || type === 'application/ogg+opus') {
stype = 'Opus';
}
else if (type === 'audio/ogg' || type === 'application/ogg') {
else if (type === 'audio/ogg' || type === 'application/ogg' || type === 'application/ogg+vorbis' || type === 'application/x-ogg') {
stype = 'Vorbis';
}
else if (type === 'audio/mpeg' || type === 'audio/MPA' || type === 'audio/mpa-robust') {
else if (type === 'application/mp3' || type === 'audio/x-mpeg' || type === 'audio/mpeg' || type === 'audio/MPA' || type === 'audio/mpa-robust') {
stype = 'MP3';
}
else if (type === 'audio/aac' || type === 'audio/mp4') {
......@@ -323,10 +341,10 @@ function generateSubType(type) {
else if (type === 'audio/aacp') {
stype = 'AAC+';
}
else if (type === 'video/webm') {
else if (type === 'video/webm' || type === 'audio/webm') {
stype = 'WebM';
}
else if (type === 'video/ogg') {
else if (type === 'video/ogg' || type === 'application/ogg+theora') {
stype = 'Theora';
}
else if (type === 'video/nsv') {
......
......@@ -2,17 +2,6 @@ DROP EXTENSION IF EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
DROP TABLE IF EXISTS server_mounts;
DROP TABLE IF EXISTS streams;
CREATE TABLE IF NOT EXISTS server_mounts (
sid uuid NOT NULL,
stream_id INTEGER REFERENCES streams(id),
listenurl VARCHAR DEFAULT NULL,
listeners INTEGER DEFAULT NULL,
max_listeners INTEGER DEFAULT NULL,
songname VARCHAR DEFAULT NULL,
lasttouch TIMESTAMP NOT NULL,
PRIMARY KEY (sid),
UNIQUE(listenurl)
);
CREATE TABLE IF NOT EXISTS streams (
id serial NOT NULL,
......@@ -34,3 +23,16 @@ CREATE TABLE IF NOT EXISTS streams (
PRIMARY KEY (id),
UNIQUE(stream_name, bitrate, codec_sub_types)
);
CREATE TABLE IF NOT EXISTS server_mounts (
sid uuid NOT NULL,
stream_id INTEGER REFERENCES streams(id),
listenurl VARCHAR DEFAULT NULL,
listeners INTEGER DEFAULT 0,
max_listeners INTEGER DEFAULT 0,
songname VARCHAR DEFAULT NULL,
lasttouch TIMESTAMP NOT NULL,
PRIMARY KEY (sid),
UNIQUE(listenurl)
);
......@@ -15,7 +15,12 @@
"swig": "1.4.2",
"validator":"*",
"xmlbuilder":"*",
"apidoc":"*"
"apidoc":"*",
"mocha":"*",
"should":"*",
"supertest":"*",
"request":"*",
"bunyan":"*"
},
"apidoc": {
"name": "Icecast Stream Directory API",
......@@ -28,5 +33,8 @@
"Genres",
"Formats"
]
},
"scripts":{
"test":"mocha"
}
}
var http = require('http')
var querystring = require("querystring");
var request = require('request');
var conf = require('konphyg')(__dirname + '/../config');
var query = require("pg-query");
var config = conf.all().config;
var async = require('async');
query.connectionParameters = config.db;
/* Thanks stackoverflow */
function makeid()
{
var text = "";
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for( var i=0; i < 10; i++ )
text += possible.charAt(Math.floor(Math.random() * possible.length));
return text;
}
var baseUrl = "http://localhost:3000/cgi-bin/yp-cgi?";
function insertStream(name, type, genre, bitrate, listen_url, desc, url, stype, cluster_pass, callback) {
params = {"action":"add","sn":name,"type":type,"genre":genre,"b":bitrate,"listenurl":listen_url};
if(desc) {
params.desc = desc;
}
if(url) {
params.url = url;
}
if(stype) {
params.stype = stype;
}
if(cluster_pass) {
params.cpswd = cluster_pass;
}
request.post({"url":baseUrl, form:params}, function(error, res, body) {
if (error || !res) {
console.log("failure");
callback(1);
return;
}
if(!res.headers) {
console.log("failure");
callback(1);
return;
}
if(res.headers.ypresponse && res.headers.ypresponse == 1) {
callback(0, res.headers.sid);
}
else {
console.log("unsuccesfully added");
callback(1);
return;
}
callback(0, res.headers.sid);
});
}
function touchStream(sid, song_title, listeners, max_listeners) {
params = {"action":"touch","sid":sid};
if(song_title) {
params.st = song_title;
}
if(listeners) {
params.listeners = listeners;
}
if(max_listeners) {
params.max_listeners = max_listeners;
}
request.post({"url":baseUrl, form:params}, function(error, res, body) {
if (error || !res) {
console.log("touch failed");
return;
}
if(res.headers.ypresponse && res.headers.ypresponse == 1) {
console.log("touched");
}
var listeners = Math.floor(Math.random()*10);
var max_modifier = Math.floor(Math.random()*5)+1;
function runtouch()
{
touchStream(sid, makeid(), listeners, listeners*max_modifier);
};
setTimeout(runtouch, (Math.random()*15000 + 5000));
});
}
function deleteStream(sid) {
params = {"action":"remove","sid":sid};
request.post({"url":baseUrl, form:params}, function(error, res, body) {
if (error) {
return;
}
if(res.headers.ypresponse && res.headers.ypresponse == 1) {
console.log("succesfully removed");
}
});
}
stream_types = ['audio/mpeg', 'application/ogg+vorbis','audio/aac',
'audio/aacp', 'application/ogg+theora','video/nsv'];