diff --git a/mtsenliven.py b/mtsenliven.py index 61e536b..7b2e1a2 100644 --- a/mtsenliven.py +++ b/mtsenliven.py @@ -57,10 +57,10 @@ except: " minetest-server package or compiling from git instructions" " on minetest.net") exit(1) -msg_flags = ["WARNING[Server]: ", "ACTION[Server]: "] -msg_lists = {} # where flag is key -for flag in msg_flags: - msg_lists[flag] = [] +msgprefix_flags = ["WARNING[Server]: ", "ACTION[Server]: "] +msgprefix_lists = {} # where flag is key +for flag in msgprefix_flags: + msgprefix_lists[flag] = [] # see https://www.endpoint.com/blog/2015/01/28/getting-realtime-output- # using-python @@ -91,7 +91,7 @@ def print_unique_only(output, err_flag=False): if flag in output: always_show_enable = True if not always_show_enable: - for flag in msg_flags: + for flag in msgprefix_flags: # such as '2018-02-06 21:08:06: WARNING[Server]: Deprecated call to get_look_yaw, use get_look_horizontal instead' # or 2018-02-06 21:08:05: ACTION[Server]: [playereffects] Wrote playereffects data into /home/owner/.minetest/worlds/FCAGameAWorld/playereffects.mt. f_i = output.find(flag) @@ -105,10 +105,10 @@ def print_unique_only(output, err_flag=False): sub_msg = wrap["opener"] + "..." + wrap["closer"] msg_msg = "similar messages" break - if sub_msg in msg_lists[found_flag]: + if sub_msg in msgprefix_lists[found_flag]: show_enable = False else: - msg_lists[found_flag].append(sub_msg) + msgprefix_lists[found_flag].append(sub_msg) if show_enable: print(output_strip) if found_flag is not None: diff --git a/webapp/README.md b/webapp/README.md index 34720ee..e28ddb9 100755 --- a/webapp/README.md +++ b/webapp/README.md @@ -10,6 +10,7 @@ see =0.10.0" + }, + "homepage": "https://github.com/nacholibre/node-readlines#readme", + "keywords": [ + "read", + "line", + "reader", + "linereader", + "readfile", + "linebyline", + "synchronous", + "sync", + "readline", + "readlines" + ], + "license": "MIT", + "main": "./readlines.js", + "name": "n-readlines", + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/nacholibre/node-readlines.git" + }, + "scripts": { + "test": "./node_modules/mocha/bin/mocha *_test.js" + }, + "version": "0.2.8" +} diff --git a/webapp/node_modules/n-readlines/readlines.js b/webapp/node_modules/n-readlines/readlines.js new file mode 100644 index 0000000..c1d70a6 --- /dev/null +++ b/webapp/node_modules/n-readlines/readlines.js @@ -0,0 +1,160 @@ +'use strict'; + +var fs = require('fs'); + +function LineByLine(file, options) { + options = options || {}; + + if (!options.readChunk) { + options.readChunk = 1024; + } + + if (!options.newLineCharacter) { + options.newLineCharacter = 0x0a; //linux line ending + } else { + options.newLineCharacter = options.newLineCharacter.charCodeAt(0); + } + + if (typeof file === 'number') { + this.fd = file; + } else { + this.fd = fs.openSync(file, 'r'); + } + + this.options = options; + + this.newLineCharacter = options.newLineCharacter; + + this.reset(); +} + +LineByLine.prototype._searchInBuffer = function(buffer, hexNeedle) { + var found = -1; + + for (var i = 0; i <= buffer.length; i++) { + var b_byte = buffer[i]; + if (b_byte === hexNeedle) { + found = i; + break; + } + } + + return found; +}; + +LineByLine.prototype.reset = function() { + this.bufferData = null; + this.bytesRead = 0; + + this.bufferPosition = 0; + this.eofReached = false; + + this.line = ''; + + this.linesCache = []; + + this.lastBytePosition = null; + + this.fdPosition = 0; +}; + +LineByLine.prototype._extractLines = function(buffer) { + var line; + var lines = []; + var bufferPosition = 0; + + var lastNewLineBufferPosition = 0; + while (true) { + var bufferPositionValue = buffer[bufferPosition++]; + + if (bufferPositionValue === this.newLineCharacter) { + line = buffer.slice(lastNewLineBufferPosition, bufferPosition); + lines.push(line); + lastNewLineBufferPosition = bufferPosition; + } else if (!bufferPositionValue) { + break; + } + } + + var leftovers = buffer.slice(lastNewLineBufferPosition, bufferPosition); + if (leftovers.length) { + lines.push(leftovers); + } + + return lines; +}; + +LineByLine.prototype._readChunk = function(lineLeftovers) { + var totalBytesRead = 0; + + var bytesRead; + var buffers = []; + do { + var readBuffer = new Buffer(this.options.readChunk); + + bytesRead = fs.readSync(this.fd, readBuffer, 0, this.options.readChunk, this.fdPosition); + totalBytesRead = totalBytesRead + bytesRead; + + this.fdPosition = this.fdPosition + bytesRead; + + buffers.push(readBuffer); + } while (bytesRead && this._searchInBuffer(buffers[buffers.length-1], this.options.newLineCharacter) === -1); + + var bufferData = Buffer.concat(buffers); + + if (bytesRead < this.options.readChunk) { + this.eofReached = true; + bufferData = bufferData.slice(0, totalBytesRead); + } + + if (bytesRead) { + this.linesCache = this._extractLines(bufferData); + + if (lineLeftovers) { + this.linesCache[0] = Buffer.concat([lineLeftovers, this.linesCache[0]]); + } + } + + return totalBytesRead; +}; + +LineByLine.prototype.next = function() { + var line = false; + + if (this.eofReached && this.linesCache.length === 0) { + return line; + } + + var bytesRead; + + if (!this.linesCache.length) { + bytesRead = this._readChunk(); + } + + if (this.linesCache.length) { + line = this.linesCache.shift(); + + var lastLineCharacter = line[line.length-1]; + + if (lastLineCharacter !== 0x0a) { + bytesRead = this._readChunk(line); + + if (bytesRead) { + line = this.linesCache.shift(); + } + } + } + + if (this.eofReached && this.linesCache.length === 0) { + fs.closeSync(this.fd); + this.fd = null; + } + + if (line && line[line.length-1] === this.newLineCharacter) { + line = line.slice(0, line.length-1); + } + + return line; +}; + +module.exports = LineByLine; diff --git a/webapp/node_modules/n-readlines/readlines_test.js b/webapp/node_modules/n-readlines/readlines_test.js new file mode 100644 index 0000000..073255d --- /dev/null +++ b/webapp/node_modules/n-readlines/readlines_test.js @@ -0,0 +1,106 @@ +'use strict'; + +var lineByLine = require('./readlines.js'); + +var assert = require('assert'); + +describe('Line by line', function() { + it('should get all lines', function () { + var filename = __dirname + '/dummy_files/twoLineFile.txt'; + + var liner = new lineByLine(filename); + + assert(liner.next().toString('ascii') === 'hello'); + assert(liner.next().toString('ascii') === 'hello2'); + assert(liner.next() === false); + assert(liner.next() === false); + assert(liner.fd === null); + }); + + it('should get all lines even if the file doesnt end with new line', function () { + var filename = __dirname + '/dummy_files/badEndFile.txt'; + + var liner = new lineByLine(filename); + + assert(liner.next().toString('ascii') === 'google.com'); + assert(liner.next().toString('ascii') === 'yahoo.com'); + assert(liner.next() === false); + assert(liner.fd === null); + }); + + it('should get all lines if there is no new lines', function () { + var filename = __dirname + '/dummy_files/noNewLinesFile.txt'; + + var liner = new lineByLine(filename); + + assert(liner.next().toString('ascii') === 'no new line'); + assert(liner.next() === false); + assert(liner.fd === null); + }); + + it('should handle empty files', function () { + var filename = __dirname + '/dummy_files/emptyFile.txt'; + + var liner = new lineByLine(filename); + + assert(liner.next() === false); + assert(liner.fd === null); + }); + + it('should read right between two chunks', function () { + var filename = __dirname + '/dummy_files/normalFile.txt'; + var liner = new lineByLine(filename, {'readChunk': 16}); + + assert(liner.next().toString('ascii') === 'google.com'); + assert(liner.next().toString('ascii') === 'yahoo.com'); + assert(liner.next().toString('ascii') === 'yandex.ru'); + assert(liner.next() === false); + assert(liner.fd === null); + }); + + it('should read empty lines', function () { + var filename = __dirname + '/dummy_files/withEmptyLines.txt'; + var liner = new lineByLine(filename); + + assert(liner.next().toString('ascii') === 'hello'); + assert(liner.next().toString('ascii') === 'hello4'); + assert(liner.next().toString('ascii') === ''); + assert(liner.next().toString('ascii') === 'hello2'); + assert(liner.next().toString('ascii') === 'hello3'); + assert(liner.next() === false); + assert(liner.fd === null); + }); + + it('should reset and start from the beggining', function() { + var filename = __dirname + '/dummy_files/normalFile.txt'; + var liner = new lineByLine(filename, {'readChunk': 16}); + + assert(liner.next().toString('ascii') === 'google.com'); + assert(liner.next().toString('ascii') === 'yahoo.com'); + + liner.reset() + + assert(liner.next().toString('ascii') === 'google.com'); + assert(liner.next().toString('ascii') === 'yahoo.com'); + assert(liner.next().toString('ascii') === 'yandex.ru'); + assert(liner.next() === false); + assert(liner.fd === null); + }); + + it('should read big lines', function() { + var filename = __dirname + '/dummy_files/bigLines.json'; + var liner = new lineByLine(filename); + + var parsedLine = JSON.parse(liner.next().toString('ascii')); + assert(parsedLine); + + var parsedLine = JSON.parse(liner.next().toString('ascii')); + assert(parsedLine); + + var parsedLine = JSON.parse(liner.next().toString('ascii')); + assert(parsedLine); + + assert(liner.next() === false); + assert(liner.fd === null); + }); +}); diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 01be6d8..705d298 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -232,6 +232,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "n-readlines": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/n-readlines/-/n-readlines-0.2.8.tgz", + "integrity": "sha512-FRr6GU0vooiPAuHMBt/Pspm4htItJKCehs8Q2urdUuigXsdQzP8V03UaQedeCmiygqJwtuGhnidWHRHhUOAg9w==" + }, "negotiator": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", diff --git a/webapp/package.json b/webapp/package.json index 19e2cab..d3fef43 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -13,6 +13,7 @@ "dependencies": { "body-parser": "^1.18.2", "cookie-parser": "^1.4.3", - "express": "^4.7.2" + "express": "^4.7.2", + "n-readlines": "^0.2.8" } } diff --git a/webapp/server.js b/webapp/server.js index a9614db..f4af7c2 100644 --- a/webapp/server.js +++ b/webapp/server.js @@ -1,25 +1,34 @@ 'use strict'; +var tz_offset = 240; //subtract this from server time to get local time; 4hrs is 240; 5hrs is 300 +//TODO: handle tz_offset not divisible by 60 +//var selected_date_s = null; +//selected_date_s = "2018-05-08"; + + var express = require('express'), //var exphbs = require("express-handlebars"); // exphbs = require('../../'); // "express-handlebars" cookieParser = require('cookie-parser'), bodyParser = require('body-parser'), //session = require('express-session'), - fs = require('fs'); + fs = require('fs'), + readlines = require('n-readlines'); const os = require('os'); var app = express(); //app.engine('handlebars', exphbs({defaultLayout: 'main'})); //app.set('view engine', 'handlebars'); var players = []; +var player_indices = {}; //see https://developer.mozilla.org/en-US/docs/Learn/Server-side/Express_Nodejs/Introduction var prev_line = null; //#region derived from mtsenliven.py -var msg_flags = ["WARNING[Server]: ", "ACTION[Server]: "] -var msg_lists = {} // where flag is key -for (flag in msg_flags) { - msg_lists[flag] = []; +var msgprefix_flags = ["WARNING[Server]: ", "ACTION[Server]: "] +var msgprefix_lists = {} // where flag is key +var mf_len = msgprefix_flags.length; +for (var mf_i=0; mf_i -1) { + msgprefix_number = mf_i; + msgprefix = msgprefix_flags[mf_i]; + break; + } + } + var skip_date_enable = false; + for (var uf_i=0; uf_i -1) { + verb_number = uf_i; + verb = unique_flags[uf_i]; + date_s = line.substring(0,10).trim(); + if (selected_date_s==null || selected_date_s==date_s) { + //console.log("(verbose message in process_logline) using '" + date_s + "' since selected '"+selected_date_s+"'"); + time_s = line.substring(time_start_i, time_start_i+8); + //console.log("using time "+time_s); + if (msgprefix!=null) { + player_name = line.substring(msgprefix_i+msgprefix.length, verb_i).trim(); + var ip_flag = " ["; + var ip_i = player_name.indexOf(ip_flag); + if (ip_i > -1) { + player_ip = player_name.substring(ip_i+ip_flag.length, player_name.length-1); + player_name = player_name.substring(0,ip_i); + } + } + else { + player_name = "<missing msgprefix&rt;"; + } + } + else { + skip_date_enable = true; + //console.log("WARNING in process_logline: skipping '" + date_s + "' since not '"+selected_date_s+"'"); + } + break; + } + } + var index = -1; // player index + if (player_name != null) { + if (player_name.length > 0) { + if (player_indices.hasOwnProperty(player_name)) { + index = player_indices[player_name]; + index_msg = "cached "; + } + else { + index = players.length; + //players.push({}); + players[index] = {}; + players[index].display_name = player_name; + player_indices[player_name] = index; + //console.log("created new index "+index); + } } else { - index = player_count; - players[index] = {}; - players[index].display_name = player_name; - player_count++; + console.log("WARNING in process_logline: zero-length player name"); } - + } + if (index<0 && !skip_date_enable) { + //console.log("(verbose message in process_logline) "+index_msg+"index was '"+index+"' but date was good '" + date_s + "' for '"+line+"'"); } if (verb == "leaves game") { - players[index].logout_time = ""; + if (index > -1) { + players[index].logout_time = time_s; + } } else if (verb == "joins game") { - players[index].login_time = ""; - } + if (index > -1) { + //console.log("using index " + index); + players[index].login_time = time_s; + } + } } -function relog_unique_only(output, err_flag=false) { +function store_unique_log_data(output, line_number, err_flag=false) { var ret = ""; var output_strip = output.trim(); var u_prefix = "active block modifiers took "; @@ -76,13 +151,13 @@ function relog_unique_only(output, err_flag=false) { } } if (!always_show_enable) { - var mf_len = msg_flags.length; + var mf_len = msgprefix_flags.length; for (var mf_i=0; mf_i= 0) { - found_flag = msg_flags[mf_i]; + found_flag = msgprefix_flags[mf_i]; break; } } @@ -98,11 +173,11 @@ function relog_unique_only(output, err_flag=false) { break; } } - if (msg_lists[found_flag].indexOf(sub_msg) > -1) { + if (msgprefix_lists[found_flag].indexOf(sub_msg) > -1) { show_enable = false; } else { - msg_lists[found_flag].push(sub_msg); + msgprefix_lists[found_flag].push(sub_msg); } } } @@ -114,9 +189,10 @@ function relog_unique_only(output, err_flag=false) { } return ret; } - +var cached_date = null; app.get('/', function (req, res) { var ret = ""; + ret += ''; // Whenever server starts the following is logged // (see also etc/example-input.txt): // @@ -124,11 +200,47 @@ app.get('/', function (req, res) { // Separator //------------- // - ret += os.homedir(); + var selected_date_s = null; + if (req.query.date) selected_date_s = req.query.date + ret += "assuming minetestserver ran as: " + os.homedir() + "
\n"; + ret += "timezone (tz_offset/60*-1): " + (Math.floor(tz_offset/60)*-1) + "
\n"; + ret += "date (this YYYY-MM-DD filter limits all log processing): " + selected_date_s + "
\n"; + if (selected_date_s==null) { + ret += '2018-05-08'; + } + if (cached_date!=selected_date_s && selected_date_s!=null) { + cached_date = selected_date_s; + players = []; + player_indices = []; + var log_paths = [os.homedir() + "/.minetest/debug_archived/2018/05/08.txt", os.homedir() + "/.minetest/debug.txt"]; + var lp_len = log_paths.length; + for (var lp_i=0; lp_i + ret += ` Your browser does not support the canvas element.
loading...
@@ -159,14 +271,8 @@ stats_html += '
' + time_area_width + "px + " + player_name_width + "px play var outputE = document.getElementById("outputArea"); outputE.innerHTML = stats_html;`; - + var arrayLength = players.length; - player_count = 0; - player_indices = {}; - var index = -1; - for () { - process_logline(line); - } for (var i = 0; i < arrayLength; i++) { ret += 'players[' + i + '] = {};' + "\n"; ret += 'players[' + i + '].login_time = "' + players[i].login_time + '";' + "\n"; @@ -179,8 +285,9 @@ var ctx = canvas.getContext("2d"); ctx.font = text_size+"px Arial"; var arrayLength = players.length; var text_x; -var index = -1; -for (var i = 0; index < arrayLength; i++) { +var index = -1;` + ret += 'var tz_offset = ' + tz_offset + ';'; + ret += `for (var i = 0; index < arrayLength; i++) { text_x = text_start_x; ctx.fillStyle = "#dce5ea"; ctx.fillRect(0, border_y+1.0, player_name_width, row_y_offset-1.0); @@ -189,7 +296,7 @@ for (var i = 0; index < arrayLength; i++) { ctx.moveTo(0, border_y); ctx.strokeStyle = "#000000"; ctx.lineTo(canvas_w, border_y); - + //ctx.stroke(); ctx.fillStyle = "#000000"; if (index==-1) { @@ -197,7 +304,7 @@ for (var i = 0; index < arrayLength; i++) { ctx.fillText("Name", text_x, text_y); ctx.font = text_size+"px Arial"; text_x = player_name_width+0.5; - + //ctx.fillStyle = "#000000"; ctx.strokeStyle = "#008000"; for (var hour=0; hour=3) { + login_places[0] -= Math.floor(tz_offset/60); + } var logout_places = players[index].logout_time.split(':'); + if (logout_places.length>=3) { + logout_places[0] -= Math.floor(tz_offset/60); + } var login_second = (+login_places[0]) * 60 * 60 + (+login_places[1]) * 60 + (+login_places[2]); var login_m = login_second / 60.0; var logout_second = (+logout_places[0]) * 60 * 60 + (+logout_places[1]) * 60 + (+logout_places[2]); @@ -236,16 +349,16 @@ for (var i = 0; index < arrayLength; i++) { index++; } `; + ret += ''; res.send(ret); //res.render('home'); }); + + var server = app.listen(3000, function () { //console.log('express-handlebars example server listening on: 3000'); var host = server.address().address; var port = server.address().port; - var this_log_path = os.homedir() + "/.minetest/debug_archived/2018/05/08.txt"; - console.log("EnlivenMinetest webapp reading '" + this_log_path + "'..."); - console.log("EnlivenMinetest webapp listening at http://%s:%s", host, port); });