diff --git a/README.md b/README.md index c6ecdf1..fc332fc 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,63 @@ Explanation for each field: "host": "127.0.0.1", "port": 6379 } + +/* Monitoring RPC services. Statistics will be displayed in Admin panel */ +"monitoring": { + "daemon": { + "checkInterval": 60, //interval of sending rpcMethod request + "rpcMethod": "getblockcount" //RPC method name + }, + "wallet": { + "checkInterval": 60, + "rpcMethod": "getbalance" + } + +/* Collect pool statistics to display in frontend charts */ +"charts": { + "pool": { + "hashrate": { + "enabled": true, //enable data collection and chart displaying in frontend + "updateInterval": 60, //how often to get current value + "stepInterval": 1800, //chart step interval calculated as average of all updated values + "maximumPeriod": 86400 //chart maximum periods (chart points number = maximumPeriod / stepInterval = 48) + }, + "workers": { + "enabled": true, + "updateInterval": 60, + "stepInterval": 1800, //chart step interval calculated as maximum of all updated values + "maximumPeriod": 86400 + }, + "difficulty": { + "enabled": true, + "updateInterval": 1800, + "stepInterval": 10800, + "maximumPeriod": 604800 + }, + "price": { //USD price of one currency coin received from cryptonator.com/api + "enabled": true, + "updateInterval": 1800, + "stepInterval": 10800, + "maximumPeriod": 604800 + }, + "profit": { //Reward * Rate / Difficulty + "enabled": true, + "updateInterval": 1800, + "stepInterval": 10800, + "maximumPeriod": 604800 + } + }, + "user": { //chart data displayed in user stats block + "hashrate": { + "enabled": true, + "updateInterval": 180, + "stepInterval": 1800, + "maximumPeriod": 86400 + }, + "payments": { //payment chart uses all user payments data stored in DB + "enabled": true + } + } ``` #### 3) [Optional] Configure cryptonote-easy-miner for your pool diff --git a/config.json b/config.json index 87c5a5d..0e62a85 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,7 @@ { "coin": "monero", "symbol": "XMR", + "coinUnits": 1000000000000, "logging": { "files": { @@ -113,5 +114,62 @@ "redis": { "host": "127.0.0.1", "port": 6379 + }, + + "monitoring": { + "daemon": { + "checkInterval": 60, + "rpcMethod": "getblockcount" + }, + "wallet": { + "checkInterval": 60, + "rpcMethod": "getbalance" + } + }, + + "charts": { + "pool": { + "hashrate": { + "enabled": true, + "updateInterval": 60, + "stepInterval": 1800, + "maximumPeriod": 86400 + }, + "workers": { + "enabled": true, + "updateInterval": 60, + "stepInterval": 1800, + "maximumPeriod": 86400 + }, + "difficulty": { + "enabled": true, + "updateInterval": 1800, + "stepInterval": 10800, + "maximumPeriod": 604800 + }, + "price": { + "enabled": true, + "updateInterval": 1800, + "stepInterval": 10800, + "maximumPeriod": 604800 + }, + "profit": { + "enabled": true, + "updateInterval": 1800, + "stepInterval": 10800, + "maximumPeriod": 604800 + } + }, + "user": { + "hashrate": { + "enabled": true, + "updateInterval": 180, + "stepInterval": 1800, + "maximumPeriod": 86400 + }, + "payments": { + "enabled": true + } + } } } diff --git a/init.js b/init.js index e13596a..93a3982 100644 --- a/init.js +++ b/init.js @@ -30,6 +30,10 @@ if (cluster.isWorker){ case 'cli': require('./lib/cli.js'); break + case 'chartsDataCollector': + require('./lib/chartsDataCollector.js'); + break + } return; } @@ -40,7 +44,7 @@ require('./lib/exceptionWriter.js')(logSystem); var singleModule = (function(){ - var validModules = ['pool', 'api', 'unlocker', 'payments']; + var validModules = ['pool', 'api', 'unlocker', 'payments', 'chartsDataCollector']; for (var i = 0; i < process.argv.length; i++){ if (process.argv[i].indexOf('-module=') === 0){ @@ -75,6 +79,9 @@ var singleModule = (function(){ case 'api': spawnApi(); break; + case 'chartsDataCollector': + spawnChartsDataCollector(); + break; } } else{ @@ -82,6 +89,7 @@ var singleModule = (function(){ spawnBlockUnlocker(); spawnPaymentProcessor(); spawnApi(); + spawnChartsDataCollector(); } spawnCli(); @@ -228,4 +236,18 @@ function spawnApi(){ function spawnCli(){ -} \ No newline at end of file +} + +function spawnChartsDataCollector(){ + if (!config.charts) return; + + var worker = cluster.fork({ + workerType: 'chartsDataCollector' + }); + worker.on('exit', function(code, signal){ + log('error', logSystem, 'chartsDataCollector died, spawning replacement...'); + setTimeout(function(){ + spawnChartsDataCollector(); + }, 2000); + }); +} diff --git a/lib/api.js b/lib/api.js index 7bc627c..93b646f 100644 --- a/lib/api.js +++ b/lib/api.js @@ -6,6 +6,8 @@ var zlib = require('zlib'); var async = require('async'); var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet); +var charts = require('./charts.js'); +var authSid = Math.round(Math.random() * 10000000000) + '' + Math.round(Math.random() * 10000000000); var logSystem = 'api'; require('./exceptionWriter.js')(logSystem); @@ -28,6 +30,7 @@ var currentStats = ""; var currentStatsCompressed = ""; var minerStats = {}; +var minersHashrate = {}; var liveConnections = {}; var addressConnections = {}; @@ -67,18 +70,20 @@ function collectStats(){ var hashrates = replies[1]; minerStats = {}; + minersHashrate = {}; for (var i = 0; i < hashrates.length; i++){ var hashParts = hashrates[i].split(':'); - minerStats[hashParts[1]] = (minerStats[hashParts[1]] || 0) + parseInt(hashParts[0]); + minersHashrate[hashParts[1]] = (minerStats[hashParts[1]] || 0) + parseInt(hashParts[0]); } var totalShares = 0; - for (var miner in minerStats){ - var shares = minerStats[miner]; + for (var miner in minersHashrate){ + var shares = minersHashrate[miner]; totalShares += shares; - minerStats[miner] = getReadableHashRateString(shares / config.api.hashrateWindow); + minersHashrate[miner] = Math.round(shares / config.api.hashrateWindow); + minerStats[miner] = getReadableHashRateString(minersHashrate[miner]); } data.miners = Object.keys(minerStats).length; @@ -133,7 +138,8 @@ function collectStats(){ minPaymentThreshold: config.payments.minPayment, denominationUnit: config.payments.denomination }); - } + }, + charts: charts.getPoolChartsData }, function(error, results){ log('info', logSystem, 'Stat collection finished: %d ms redis, %d ms daemon', [redisFinished - startTime, daemonFinished - startTime]); @@ -237,7 +243,13 @@ function handleMinerStats(urlParts, response){ } var stats = replies[0]; stats.hashrate = minerStats[address]; - response.end(JSON.stringify({stats: stats, payments: replies[1]})); + charts.getUserChartsData(address, replies[1], function(error, chartsData) { + response.end(JSON.stringify({ + stats: stats, + payments: replies[1], + charts: chartsData + })); + }); }); } } @@ -307,25 +319,59 @@ function handleGetBlocks(urlParts, response){ }); } +function handleGetMinersHashrate(response) { + var reply = JSON.stringify({ + minersHashrate: minersHashrate + }); + response.writeHead("200", { + 'Access-Control-Allow-Origin': '*', + 'Cache-Control': 'no-cache', + 'Content-Type': 'application/json', + 'Content-Length': reply.length + }); + response.end(reply); +} -collectStats(); +function parseCookies(request) { + var list = {}, + rc = request.headers.cookie; + rc && rc.split(';').forEach(function(cookie) { + var parts = cookie.split('='); + list[parts.shift().trim()] = unescape(parts.join('=')); + }); + return list; +} function authorize(request, response){ + if(request.connection.remoteAddress == '127.0.0.1') { + return true; + } response.setHeader('Access-Control-Allow-Origin', '*'); + var cookies = parseCookies(request); + if(cookies.sid && cookies.sid == authSid) { + return true; + } + var sentPass = url.parse(request.url, true).query.password; + if (sentPass !== config.api.password){ response.statusCode = 401; response.end('invalid password'); return; } + log('warn', logSystem, 'Admin authorized'); response.statusCode = 200; + + var cookieExpire = new Date( new Date().getTime() + 60*60*24*1000); + response.setHeader('Set-Cookie', 'sid=' + authSid + '; path=/; expires=' + cookieExpire.toUTCString()); response.setHeader('Cache-Control', 'no-cache'); response.setHeader('Content-Type', 'application/json'); + return true; } @@ -406,6 +452,149 @@ function handleAdminStats(response){ } + +function handleAdminUsers(response){ + async.waterfall([ + // get workers Redis keys + function(callback) { + redisClient.keys(config.coin + ':workers:*', callback); + }, + // get workers data + function(workerKeys, callback) { + var redisCommands = workerKeys.map(function(k) { + return ['hmget', k, 'balance', 'paid', 'lastShare', 'hashes']; + }); + redisClient.multi(redisCommands).exec(function(error, redisData) { + var workersData = {}; + var addressLength = config.poolServer.poolAddress.length; + for(var i in redisData) { + var address = workerKeys[i].substr(-addressLength); + var data = redisData[i]; + workersData[address] = { + pending: data[0] / config.coinUnits, + paid: data[1] / config.coinUnits, + lastShare: data[2], + hashes: data[3], + hashrate: minersHashrate[address] ? minersHashrate[address] : 0 + }; + } + callback(null, workersData); + }); + } + ], function(error, workersData) { + if(error) { + response.end(JSON.stringify({error: 'error collecting users stats'})); + return; + } + response.end(JSON.stringify(workersData)); + } + ); +} + + +function handleAdminMonitoring(response) { + response.writeHead("200", { + 'Access-Control-Allow-Origin': '*', + 'Cache-Control': 'no-cache', + 'Content-Type': 'application/json' + }); + async.parallel({ + monitoring: getMonitoringData, + logs: getLogFiles + }, function(error, result) { + response.end(JSON.stringify(result)); + }); +} + +function handleAdminLog(urlParts, response){ + var file = urlParts.query.file; + var filePath = config.logging.files.directory + '/' + file; + if(!file.match(/^\w+\.log$/)) { + response.end('wrong log file'); + } + response.writeHead(200, { + 'Content-Type': 'text/plain', + 'Cache-Control': 'no-cache', + 'Content-Length': fs.statSync(filePath).size + }); + fs.createReadStream(filePath).pipe(response) +} + + +function startRpcMonitoring(rpc, module, method, interval) { + setInterval(function() { + rpc(method, {}, function(error, response) { + var stat = { + lastCheck: new Date() / 1000 | 0, + lastStatus: error ? 'fail' : 'ok', + lastResponse: JSON.stringify(error ? error : response) + }; + if(error) { + stat.lastFail = stat.lastCheck; + stat.lastFailResponse = stat.lastResponse; + } + var key = getMonitoringDataKey(module); + var redisCommands = []; + for(var property in stat) { + redisCommands.push(['hset', key, property, stat[property]]); + } + redisClient.multi(redisCommands).exec(); + }); + }, interval * 1000); +} + +function getMonitoringDataKey(module) { + return config.coin + ':status:' + module; +} + +function initMonitoring() { + var modulesRpc = { + daemon: apiInterfaces.rpcDaemon, + wallet: apiInterfaces.rpcWallet + }; + for(var module in config.monitoring) { + var settings = config.monitoring[module]; + if(settings.checkInterval) { + startRpcMonitoring(modulesRpc[module], module, settings.rpcMethod, settings.checkInterval); + } + } +} + + + +function getMonitoringData(callback) { + var modules = Object.keys(config.monitoring); + var redisCommands = []; + for(var i in modules) { + redisCommands.push(['hgetall', getMonitoringDataKey(modules[i])]) + } + redisClient.multi(redisCommands).exec(function(error, results) { + var stats = {}; + for(var i in modules) { + if(results[i]) { + stats[modules[i]] = results[i]; + } + } + callback(error, stats); + }); +} + +function getLogFiles(callback) { + var dir = config.logging.files.directory; + fs.readdir(dir, function(error, files) { + var logs = {}; + for(var i in files) { + var file = files[i]; + var stats = fs.statSync(dir + '/' + file); + logs[file] = { + size: stats.size, + changed: Date.parse(stats.mtime) / 1000 | 0 + } + } + callback(error, logs); + }); +} + var server = http.createServer(function(request, response){ if (request.method.toUpperCase() === "OPTIONS"){ @@ -426,12 +615,13 @@ var server = http.createServer(function(request, response){ switch(urlParts.pathname){ case '/stats': - var reply = currentStatsCompressed; + var deflate = request.headers['accept-encoding'] && request.headers['accept-encoding'].indexOf('deflate') != -1; + var reply = deflate ? currentStatsCompressed : currentStats; response.writeHead("200", { 'Access-Control-Allow-Origin': '*', 'Cache-Control': 'no-cache', 'Content-Type': 'application/json', - 'Content-Encoding': 'deflate', + 'Content-Encoding': deflate ? 'deflate' : '', 'Content-Length': reply.length }); response.end(reply); @@ -462,9 +652,33 @@ var server = http.createServer(function(request, response){ case '/admin_stats': if (!authorize(request, response)) return; - log('warn', logSystem, 'Admin authorized'); handleAdminStats(response); break; + case '/admin_monitoring': + if(!authorize(request, response)) { + return; + } + handleAdminMonitoring(response); + break; + case '/admin_log': + if(!authorize(request, response)) { + return; + } + handleAdminLog(urlParts, response); + break; + case '/admin_users': + if(!authorize(request, response)) { + return; + } + handleAdminUsers(response); + break; + + case '/miners_hashrate': + if (!authorize(request, response)) + return; + handleGetMinersHashrate(response); + break; + default: response.writeHead(404, { 'Access-Control-Allow-Origin': '*' @@ -472,11 +686,11 @@ var server = http.createServer(function(request, response){ response.end('Invalid API call'); break; } - - }); +collectStats(); +initMonitoring(); server.listen(config.api.port, function(){ log('info', logSystem, 'API started & listening on port %d', [config.api.port]); -}); \ No newline at end of file +}); diff --git a/lib/apiInterfaces.js b/lib/apiInterfaces.js index 6b89170..f5948e2 100644 --- a/lib/apiInterfaces.js +++ b/lib/apiInterfaces.js @@ -1,12 +1,14 @@ var http = require('http'); +var https = require('https'); -function jsonHttpRequest(host, port, data, callback){ +function jsonHttpRequest(host, port, data, callback, path){ + path = path || '/json_rpc'; var options = { hostname: host, port: port, - path: '/json_rpc', - method: 'POST', + path: path, + method: data ? 'POST' : 'GET', headers: { 'Content-Length': data.length, 'Content-Type': 'application/json', @@ -14,7 +16,7 @@ function jsonHttpRequest(host, port, data, callback){ } }; - var req = http.request(options, function(res){ + var req = (port == 443 ? https : http).request(options, function(res){ var replyData = ''; res.setEncoding('utf8'); res.on('data', function(chunk){ @@ -72,7 +74,7 @@ function batchRpc(host, port, array, callback){ } -module.exports = function(daemonConfig, walletConfig){ +module.exports = function(daemonConfig, walletConfig, poolApiConfig){ return { batchRpcDaemon: function(batchArray, callback){ batchRpc(daemonConfig.host, daemonConfig.port, batchArray, callback); @@ -82,6 +84,10 @@ module.exports = function(daemonConfig, walletConfig){ }, rpcWallet: function(method, params, callback){ rpc(walletConfig.host, walletConfig.port, method, params, callback); - } + }, + pool: function(method, callback){ + jsonHttpRequest('127.0.0.1', poolApiConfig.port, '', callback, method); + }, + jsonHttpRequest: jsonHttpRequest } -}; \ No newline at end of file +}; diff --git a/lib/blockUnlocker.js b/lib/blockUnlocker.js index fa2fe33..7f418d1 100644 --- a/lib/blockUnlocker.js +++ b/lib/blockUnlocker.js @@ -1,6 +1,6 @@ var async = require('async'); -var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet); +var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet, config.api); var logSystem = 'unlocker'; require('./exceptionWriter.js')(logSystem); diff --git a/lib/charts.js b/lib/charts.js new file mode 100644 index 0000000..9aad8fd --- /dev/null +++ b/lib/charts.js @@ -0,0 +1,240 @@ +var fs = require('fs'); +var async = require('async'); +var http = require('http'); +var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet, config.api); + +var logSystem = 'charts'; +require('./exceptionWriter.js')(logSystem); + +log('info', logSystem, 'Started'); + +function startDataCollectors() { + async.each(Object.keys(config.charts.pool), function(chartName) { + var settings = config.charts.pool[chartName]; + if(settings.enabled) { + setInterval(function() { + collectPoolStatWithInterval(chartName, settings); + }, settings.updateInterval * 1000); + } + }); + + var settings = config.charts.user.hashrate; + if(settings.enabled) { + setInterval(function() { + collectUsersHashrate('hashrate', settings); + }, settings.updateInterval * 1000) + } +} + +function getChartDataFromRedis(chartName, callback) { + redisClient.get(getStatsRedisKey(chartName), function(error, data) { + callback(data ? JSON.parse(data) : []); + }); +} + +function getUserHashrateChartData(address, callback) { + getChartDataFromRedis('hashrate:' + address, callback); +} + +function convertPaymentsDataToChart(paymentsData) { + var data = []; + if(paymentsData && paymentsData.length) { + for(var i = 0; paymentsData[i]; i += 2) { + data.push([+paymentsData[i + 1], paymentsData[i].split(':')[1] / config.coinUnits]); + } + } + return data; +} + +function getUserChartsData(address, paymentsData, callback) { + var stats = {}; + var chartsFuncs = { + hashrate: function(callback) { + getUserHashrateChartData(address, function(data) { + callback(null, data); + }); + }, + + payments: function(callback) { + callback(null, convertPaymentsDataToChart(paymentsData)); + } + }; + for(var chartName in chartsFuncs) { + if(!config.charts.user[chartName].enabled) { + delete chartsFuncs[chartName]; + } + } + async.parallel(chartsFuncs, callback); +} + +function getStatsRedisKey(chartName) { + return config.coin + ':charts:' + chartName; +} + +var chartStatFuncs = { + hashrate: getPoolHashrate, + workers: getPoolWorkers, + difficulty: getNetworkDifficulty, + price: getCoinPrice, + profit: getCoinProfit +}; + +var statValueHandler = { + avg: function(set, value) { + set[1] = (set[1] * set[2] + value) / (set[2] + 1); + }, + round: function(set, value) { + statValueHandler.avg(set, value); + set[1] = Math.round(set[1]); + }, + max: function(set, value) { + if(value > set[1]) { + set[1] = value; + } + } +}; + +var preSaveFunctions = { + hashrate: statValueHandler.avg, + workers: statValueHandler.max, + difficulty: statValueHandler.round, + price: statValueHandler.avg, + profit: statValueHandler.avg +}; + +function storeCollectedValues(chartName, values, settings) { + for(var i in values) { + storeCollectedValue(chartName + ':' + i, values[i], settings); + } +} + +function storeCollectedValue(chartName, value, settings) { + var now = new Date() / 1000 | 0; + getChartDataFromRedis(chartName, function(sets) { + var lastSet = sets[sets.length - 1]; // [time, avgValue, updatesCount] + if(!lastSet || now - lastSet[0] > settings.stepInterval) { + lastSet = [now, value, 1]; + sets.push(lastSet); + while(now - sets[0][0] > settings.maximumPeriod) { // clear old sets + sets.shift(); + } + } + else { + preSaveFunctions[chartName] + ? preSaveFunctions[chartName](lastSet, value) + : statValueHandler.round(lastSet, value); + lastSet[2]++; + } + redisClient.set(getStatsRedisKey(chartName), JSON.stringify(sets)); + log('info', logSystem, chartName + ' chart collected value ' + value + '. Total sets count ' + sets.length); + log('info', logSystem, chartName + ' data: ' + JSON.stringify(sets)); + }); +} + +function collectPoolStatWithInterval(chartName, settings) { + async.waterfall([ + chartStatFuncs[chartName], + function(value, callback) { + storeCollectedValue(chartName, value, settings, callback); + } + ]); +} + +function getPoolStats(callback) { + apiInterfaces.pool('/stats', callback); +} + +function getPoolHashrate(callback) { + getPoolStats(function(error, stats) { + callback(error, stats.pool ? Math.round(stats.pool.hashrate) : null); + }); +} + +function getPoolWorkers(callback) { + getPoolStats(function(error, stats) { + callback(error, stats.pool ? stats.pool.miners : null); + }); +} + +function getNetworkDifficulty(callback) { + getPoolStats(function(error, stats) { + callback(error, stats.pool ? stats.network.difficulty : null); + }); +} + +function getUsersHashrates(callback) { + apiInterfaces.pool('/miners_hashrate', function(error, data) { + callback(data.minersHashrate); + }); +} + +function collectUsersHashrate(chartName, settings) { + var redisBaseKey = getStatsRedisKey(chartName) + ':'; + redisClient.keys(redisBaseKey + '*', function(keys) { + var hashrates = {}; + for(var i in keys) { + hashrates[keys[i].substr(keys[i].length)] = 0; + } + getUsersHashrates(function(newHashrates) { + for(var address in newHashrates) { + hashrates[address] = newHashrates[address]; + } + storeCollectedValues(chartName, hashrates, settings); + }); + }); +} + +function getCoinPrice(callback) { + apiInterfaces.jsonHttpRequest('www.cryptonator.com', 443, '', function(error, response) { + callback(response.error ? response.error : error, response.success ? +response.ticker.price : null); + }, '/api/ticker/' + config.symbol.toLowerCase() + '-usd'); +} + +function getCoinProfit(callback) { + getCoinPrice(function(error, price) { + if(error) { + callback(error); + return; + } + getPoolStats(function(error, stats) { + if(error) { + callback(error); + return; + } + callback(null, stats.network.reward * price / stats.network.difficulty / config.coinUnits); + }); + }); +} + +function getPoolChartsData(callback) { + var chartsNames = []; + var redisKeys = []; + for(var chartName in config.charts.pool) { + if(config.charts.pool[chartName].enabled) { + chartsNames.push(chartName); + redisKeys.push(getStatsRedisKey(chartName)); + } + } + if(redisKeys.length) { + redisClient.mget(redisKeys, function(error, data) { + var stats = {}; + if(data) { + for(var i in data) { + if(data[i]) { + stats[chartsNames[i]] = JSON.parse(data[i]); + } + } + } + callback(error, stats); + }); + } + else { + callback(null, {}); + } +} + +module.exports = { + startDataCollectors: startDataCollectors, + getUserChartsData: getUserChartsData, + getPoolChartsData: getPoolChartsData +}; diff --git a/lib/chartsDataCollector.js b/lib/chartsDataCollector.js new file mode 100644 index 0000000..293aea9 --- /dev/null +++ b/lib/chartsDataCollector.js @@ -0,0 +1,11 @@ +var fs = require('fs'); +var async = require('async'); +var http = require('http'); +var charts = require('./charts.js'); + +var logSystem = 'chartsDataCollector'; +require('./exceptionWriter.js')(logSystem); + +log('info', logSystem, 'Started'); + +charts.startDataCollectors(); diff --git a/lib/paymentProcessor.js b/lib/paymentProcessor.js index 9d3c6bd..d9d0cc1 100644 --- a/lib/paymentProcessor.js +++ b/lib/paymentProcessor.js @@ -2,7 +2,7 @@ var fs = require('fs'); var async = require('async'); -var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet); +var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet, config.api); var logSystem = 'payments'; diff --git a/lib/pool.js b/lib/pool.js index 7654eeb..bf8a251 100644 --- a/lib/pool.js +++ b/lib/pool.js @@ -13,7 +13,7 @@ var threadId = '(Thread ' + process.env.forkId + ') '; var logSystem = 'pool'; require('./exceptionWriter.js')(logSystem); -var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet); +var apiInterfaces = require('./apiInterfaces.js')(config.daemon, config.wallet, config.api); var utils = require('./utils.js'); var log = function(severity, system, text, data){ diff --git a/website/admin.html b/website/admin.html index c07bf1f..a033ff2 100644 --- a/website/admin.html +++ b/website/admin.html @@ -1,155 +1,259 @@
- --