working on piping

mtsenliven.py

@@ -4,6 +4,15 @@
 
 import os
 from mtanalyze.minetestinfo import *
+try:
+    from Threading import thread as Thread # Python 2
+except:
+    from threading import Thread # Python 3
+# if sys.version[0] == '2':
+try:
+    from Queue import Queue # Python 2
+except:
+    from queue import Queue # Python 3
 import subprocess, signal
 game_id = "ENLIVEN"
 #screen -S MinetestServer $mts --gameid ENLIVEN --worldname FCAGameAWorld
@@ -35,8 +44,10 @@ try:
     process = subprocess.Popen(
         [mts, '--gameid', game_id, '--worldname', wn],
         stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE
+        stderr=subprocess.PIPE,
+        bufsize=1
     )
+    #bufsize=1 as per jfs on https://stackoverflow.com/questions/31833897/python-read-from-subprocess-stdout-and-stderr-separately-while-preserving-order
 except:
     print(mts + " could not be executed. Try installing the "
           " minetest-server package or compiling from git instructions"
@@ -49,7 +60,7 @@ for flag in msg_flags:
 # see https://www.endpoint.com/blog/2015/01/28/getting-realtime-output-
 # using-python
 
-def print_unique_only(output):
+def print_unique_only(output, err_flag=False):
     output_strip = output.strip()
     # (out_bytes is bytes)
     show_enable = True
@@ -75,24 +86,65 @@ def print_unique_only(output):
                   " time the message above will be shown")
 
 def process_msg(bstring):
-    output = bstring.decode("utf-8")
-            # works on python2 or 3
-    print_unique_only(output)
+    output = bstring
+    err_flag = False
+    try:
+        output = bstring.decode("utf-8")
+                # works on python2 or 3
+    except AttributeError:
+        output = bstring
+    if output[:1] == "<":
+        stop_s = ">: "
+        closer_i = output.find(stop_s)
+        if closer_i >= 0:
+            next_i = closer_i + len(stop_s)
+            err_flag = True
+            output = output[next_i:]
+    print_unique_only(output, err_flag=err_flag)
+
+# see jfs's answer on https://stackoverflow.com/questions/31833897/python-read-from-subprocess-stdout-and-stderr-separately-while-preserving-order
+def reader(pipe, q):
+    try:
+        with pipe:
+            for line in iter(pipe.readline, b''):
+                q.put((pipe, line))
+    finally:
+        q.put(None)
+
+q = Queue()
+Thread(target=reader, args=[process.stdout, q]).start()
+Thread(target=reader, args=[process.stderr, q]).start()
+
+for _ in range(2):
+    for source, line in iter(q.get, None):
+        # print "%s: %s" % (source, line),
+        s = source
+        l = line
+        # NOTE: source is a string such as "<_io.BufferedReader name=5>"
+        try:
+            l = line.decode("utf-8")
+        except:
+            # this should never happen but doesn't matter anyway
+            pass
+        process_msg("%s: %s" % (s, l))
 
+exit(0)
 
 while True:
     try:
-        # out_bytes = process.stdout.readline()
+        # can deadlock on high volume--use communicate instead
+        # as per https://docs.python.org/2/library/subprocess.html
+        out_bytes = process.stdout.readline()
         # err_bytes = process.stderr.readline()
-        out_bytes, err_bytes = process.communicate()
+           # (err_bytes == '') and \
         if (out_bytes == '') and \
-           (err_bytes == '') and \
            (process.poll() is not None):
             break
         if out_bytes:
             process_msg(out_bytes)
-        if err_bytes:
-            process_msg(err_bytes)
+        # if err_bytes:
+            # process_msg(err_bytes)
         rc = process.poll()
     except KeyboardInterrupt:
         break
+# process.kill()