From 4c81c2c636fb30df23803ada5495447a400f7897 Mon Sep 17 00:00:00 2001 From: poikilos <7557867+poikilos@users.noreply.github.com> Date: Mon, 14 Mar 2016 15:28:10 -0400 Subject: [PATCH] debugging decachunk and chunk generation and border chunk detection --- chunkymap-regen.py | 350 ++++++++++++++++++++++++--------------------- expertmm.py | 8 +- 2 files changed, 191 insertions(+), 167 deletions(-) diff --git a/chunkymap-regen.py b/chunkymap-regen.py index bfe36e0..7da805d 100644 --- a/chunkymap-regen.py +++ b/chunkymap-regen.py @@ -17,7 +17,7 @@ from minetestinfo import * from expertmm import * from PIL import Image, ImageDraw, ImageFont, ImageColor - +mode_to_bpp = {'1':1, 'L':8, 'P':8, 'RGB':24, 'RGBA':32, 'CMYK':32, 'YCbCr':24, 'I':32, 'F':32} #best_timer = timeit.default_timer #if sys.platform == "win32": # on Windows, the best timer is time.clock() @@ -243,6 +243,7 @@ class MTChunks: data_16px_path = None data_160px_path = None FLAG_EMPTY_HEXCOLOR = "#010000" + FLAG_COLORS_LIST = None world_name = None chunkymap_thisworld_data_path = None genresult_name_opener_string = "chunk_" @@ -251,7 +252,12 @@ class MTChunks: def __init__(self): #formerly checkpaths() in global scope #self.force_rerender_decachunks_enable = True - self.min_indent = " " + self.FLAG_COLORS_LIST = list() + self.FLAG_COLOR_CHANNELS = get_list_from_hex(self.FLAG_EMPTY_HEXCOLOR) + self.FLAG_COLORS_LIST.append(self.FLAG_COLOR_CHANNELS) + self.FLAG_COLORS_LIST.append((255,255,255)) #for compatibility with maps generated by earlier versions ONLY + self.FLAG_COLORS_LIST.append((0,0,0)) #for compatibility with maps generated by earlier versions ONLY + min_indent = " " self.decachunks = {} self.rendered_this_session_count = 0 self.is_backend_detected = False @@ -585,24 +591,27 @@ class MTChunks: #-1 becomes -10 return int(decachunky_x*10) + + def is_worldborder_chunk(self, chunky_x, chunky_z): result = False - image_path = get_chunk_image_path(chunky_x, chunky_z) + image_path = self.get_chunk_image_path(chunky_x, chunky_z) + border_pixel_count = 0 if os.path.isfile(image_path): original_im = Image.open(image_path) im = original_im - if im.bits<24: + bit_count = 24 + try: + bit_count = mode_to_bpp[im.mode] + except: + print("ERROR in is_worldborder_chunk: unknown image mode "+str(im.mode)+" so can't get bitdepth of chunk") + if bit_count<24:#if im.bits<24: im = original_im.convert('RGB') width, height = im.size pixel_count = width*height pixel_count_f = float(pixel_count) border_count = 0 - FLAG_COLORS_LIST = list() - FLAG_COLOR_CHANNELS = get_list_from_hex(self.FLAG_EMPTY_HEXCOLOR) - FLAG_COLORS_LIST.append(FLAG_COLOR_CHANNELS) - FLAG_COLORS_LIST.append((255,255,255)) #for compatibility with maps generated by earlier versions ONLY - FLAG_COLORS_LIST.append((0,0,0)) #for compatibility with maps generated by earlier versions ONLY - for FLAG_COLOR in FLAG_COLORS_LIST: + for FLAG_COLOR in self.FLAG_COLORS_LIST: if len(FLAG_COLOR)==3 or len(FLAG_COLOR)==4: for y in range(0,height): for x in range(0,width): @@ -616,167 +625,177 @@ class MTChunks: raw_input("ERROR: FLAG_COLOR (obtained from FLAG_EMPTY_HEXCOLOR) has "+len(FLAG_COLOR)+" element(s) (3 or 4 expected)") return result - def is_chunk_on_todo_list(self, chunky_pos): + def get_index_of_chunk_on_todo_list(self, chunky_pos): result = -1 - if self.todo_index -1: + if self.todo_index -1 + if is_any_part_queued: + if queued_chunk_coords is None: + queued_chunk_coords = list() + queued_chunk_coords.append(coords) + break + chunky_x += 1 if is_any_part_queued: - if queued_chunk_coords is None: - queued_chunk_coords = list() - queued_chunk_coords.append(coords) break - chunky_x += 1 - if is_any_part_queued: - break - chunky_z += 1 - chunky_offset_z += 1 - if not is_any_part_queued: - is_chunk_complete = True - - if is_chunk_complete: - ### NOTE: a chunk is incomplete if any rendered nonworldborder chunk touches a nonrendered chunk - for chunky_pos in chunky_coord_list: - this_chunky_x, this_chunky_z = chunky_pos - if not self.is_chunk_rendered_on_dest(this_chunky_x, this_chunky_z): - outline_coords_list = self.get_outline_coords_list(this_chunky_x, this_chunky_z, True) - if outline_coords_list is not None: - for nearby_chunky_pos in outline_coords_list: - nearby_chunky_x, nearby_chunky_z = nearby_chunky_pos - nearby_chunk_luid = self.get_chunk_luid(nearby_chunky_x, nearby_chunky_z) - if (nearby_chunk_luid in self.chunks and self.chunks[nearby_chunk_luid].is_fresh) or self.is_chunk_rendered_on_dest(nearby_chunky_x, nearby_chunky_z): - this_is_worldborder_chunk = False - if (nearby_chunk_luid in self.chunks and "is_worldborder" in self.chunks[nearby_chunk_luid].metadata and self.chunks[nearby_chunk_luid].metadata["is_worldborder"]): - this_is_worldborder_chunk = True - elif self.is_worldborder_chunk(nearby_chunky_x, nearby_chunky_z): - this_is_worldborder_chunk = True - self.prepare_chunk_meta(nearby_chunky_x, nearby_chunky_z) - if "is_worldborder" not in self.chunks[nearby_chunk_luid].metadata or (self.chunks[nearby_chunk_luid].metadata["is_worldborder"] != True) - self.chunks[nearby_chunk_luid].metadata["is_worldborder"] = True - self.save_chunk_meta(nearby_chunky_x, nearby_chunky_z) - if not this_is_worldborder_chunk: - #empty chunk would not touch NON-worldborder chunk if decachunk was complete - is_chunk_complete = False - break + chunky_z += 1 + chunky_offset_z += 1 + if not is_any_part_queued: + is_chunk_complete = True + unfinished_chunky_coord = None + if is_chunk_complete: + ### NOTE: a chunk is incomplete if any rendered nonworldborder chunk touches a nonrendered chunk + for chunky_pos in chunky_coord_list: + this_chunky_x, this_chunky_z = chunky_pos + if not self.is_chunk_rendered_on_dest(this_chunky_x, this_chunky_z): + outline_coords_list = self.get_outline_coords_list(this_chunky_x, this_chunky_z, True) + if outline_coords_list is not None: + for nearby_chunky_pos in outline_coords_list: + nearby_chunky_x, nearby_chunky_z = nearby_chunky_pos + nearby_chunk_luid = self.get_chunk_luid(nearby_chunky_x, nearby_chunky_z) + if (nearby_chunk_luid in self.chunks and self.chunks[nearby_chunk_luid].is_fresh) or self.is_chunk_rendered_on_dest(nearby_chunky_x, nearby_chunky_z): + this_is_worldborder_chunk = False + if (nearby_chunk_luid in self.chunks and "is_worldborder" in self.chunks[nearby_chunk_luid].metadata and self.chunks[nearby_chunk_luid].metadata["is_worldborder"]): + this_is_worldborder_chunk = True + elif self.is_worldborder_chunk(nearby_chunky_x, nearby_chunky_z): + this_is_worldborder_chunk = True + self.prepare_chunk_meta(nearby_chunky_x, nearby_chunky_z) + if ("is_worldborder" not in self.chunks[nearby_chunk_luid].metadata) or (self.chunks[nearby_chunk_luid].metadata["is_worldborder"] != True): + self.chunks[nearby_chunk_luid].metadata["is_worldborder"] = True + self.save_chunk_meta(nearby_chunky_x, nearby_chunky_z) + if not this_is_worldborder_chunk: + #empty chunk would not touch NON-worldborder chunk if decachunk was complete + is_chunk_complete = False + unfinished_chunky_coord = nearby_chunky_x, nearby_chunky_z + break + else: + print(min_indent+"ERROR in check_decachunk_containing_chunk: no outline of chunks could be found around "+str(chunky_pos)) + if not is_chunk_complete: + break + + + + #if not is_any_part_queued: + #if queued_chunk_coords is None: + if is_chunk_complete and not is_any_part_queued: + print("") + print("") + print(" Rendering 160px decachunk "+str((decachunky_x, decachunky_z))) + if self.verbose_enable: + print(" USING ("+str(len(chunky_coord_list))+") chunks (region "+str(chunky_min_x)+":"+str(chunky_max_x)+","+str(chunky_min_z)+":"+str(chunky_max_z)+"): "+str(chunky_coord_list)) + print("") + else: + print(" USING ("+str(len(chunky_coord_list))+") chunks (region "+str(chunky_min_x)+":"+str(chunky_max_x)+","+str(chunky_min_z)+":"+str(chunky_max_z)+")") + decachunk_global_coords = decachunky_x*160, decachunky_z*160 + im = Image.new("RGB", (160, 160), self.FLAG_EMPTY_HEXCOLOR) + decachunk_yaml_path = self.get_decachunk_yaml_path_from_decachunk(decachunky_x, decachunky_z) + decachunk_image_path = self.get_decachunk_image_path_from_decachunk(decachunky_x, decachunky_z) + combined_count = 0 + contains_chunk_luids = list() + + for coord in chunky_coord_list: + chunky_x, chunky_z = coord + chunky_offset_x = chunky_x - chunky_min_x + chunky_offset_z = chunky_z - chunky_min_z + chunk_image_path = self.get_chunk_image_path(chunky_x, chunky_z) + if os.path.isfile(chunk_image_path): + preview_strings[chunky_offset_z] += "1" + participle="initializing" + try: + participle="opening path" + chunk_im = Image.open(open(chunk_image_path, 'rb')) # double-open to make sure file is finished writing + #NOTE: PIL automatically closes, otherwise you can do something like https://bytes.com/topic/python/answers/24308-pil-do-i-need-close + #fp = open(file_name, "rb") + #im = Image.open(fp) # open from file object + #im.load() # make sure PIL has read the data + #fp.close() + chunk_global_coords = chunky_x*16, chunky_z*16 + chunk_local_coords = chunk_global_coords[0]-decachunk_global_coords[0], chunk_global_coords[1]-decachunk_global_coords[1] + offset = chunk_local_coords[0], 160-chunk_local_coords[1] # convert to inverted cartesian since that's the coordinate system of images + im.paste(chunk_im, offset) + contains_chunk_luids.append(self.get_chunk_luid(chunky_x, chunky_z)) + except: + print(min_indent+"Could not finish "+participle+" in check_decachunk_containing_chunk:") + view_traceback() else: - print(self.min_indent+"ERROR in check_decachunk_containing_chunk: no outline of chunks could be found around "+str(chunky_pos)) - if not is_chunk_complete: - break - - - - #if not is_any_part_queued: - #if queued_chunk_coords is None: - if is_chunk_complete and not is_any_part_queued: - print("") - print("") - print(" Rendering 160px decachunk "+str((decachunky_x, decachunky_z))) - if self.verbose_enable: - print(" USING ("+str(len(chunky_coord_list))+") chunks (region "+str(chunky_min_x)+":"+str(chunky_max_x)+","+str(chunky_min_z)+":"+str(chunky_max_z)+"): "+str(chunky_coord_list)) + preview_strings[chunky_offset_z] += "0" + chunky_offset_z = z_chunky_count - 1 + try: + print(min_indent+"Decachunk available chunk mask (height:"+str(z_chunky_count)+"):") + while chunky_offset_z>=0: + if preview_strings[chunky_offset_z] is None: + preview_strings[chunky_offset_z] = "" + print(min_indent+" "+str(chunky_offset_z)+":"+preview_strings[chunky_offset_z]) + chunky_offset_z -= 1 + except: + print(min_indent+"Could not finish showing mask (this should never happen)") + print(min_indent+" z_chunky_count:"+str(z_chunky_count)) + print(min_indent+" len(preview_strings):"+str(len(preview_strings))) + print(min_indent+" chunky_min_x:"+str(chunky_min_x)) + print(min_indent+" chunky_max_x:"+str(chunky_max_x)) + print(min_indent+" chunky_min_z:"+str(chunky_min_z)) + print(min_indent+" chunky_max_z:"+str(chunky_max_z)) + view_traceback() print("") - else: - print(" USING ("+str(len(chunky_coord_list))+") chunks (region "+str(chunky_min_x)+":"+str(chunky_max_x)+","+str(chunky_min_z)+":"+str(chunky_max_z)+")") - decachunk_global_coords = decachunky_x*160, decachunky_z*160 - im = Image.new("RGB", (160, 160), self.FLAG_EMPTY_HEXCOLOR) - decachunk_yaml_path = self.get_decachunk_yaml_path_from_decachunk(decachunky_x, decachunky_z) - decachunk_image_path = self.get_decachunk_image_path_from_decachunk(decachunky_x, decachunky_z) - combined_count = 0 - contains_chunk_luids = list() - - for coord in chunky_coord_list: - chunky_x, chunky_z = coord - chunky_offset_x = chunky_x - chunky_min_x - chunky_offset_z = chunky_z - chunky_min_z - chunk_image_path = self.get_chunk_image_path(chunky_x, chunky_z) - if os.path.isfile(chunk_image_path): - preview_strings[chunky_offset_z] += "1" - participle="initializing" - try: - participle="opening path" - chunk_im = Image.open(open(chunk_image_path, 'rb')) # double-open to make sure file is finished writing - #NOTE: PIL automatically closes, otherwise you can do something like https://bytes.com/topic/python/answers/24308-pil-do-i-need-close - #fp = open(file_name, "rb") - #im = Image.open(fp) # open from file object - #im.load() # make sure PIL has read the data - #fp.close() - chunk_global_coords = chunky_x*16, chunky_z*16 - chunk_local_coords = chunk_global_coords[0]-decachunk_global_coords[0], chunk_global_coords[1]-decachunk_global_coords[1] - offset = chunk_local_coords[0], 160-chunk_local_coords[1] # convert to inverted cartesian since that's the coordinate system of images - im.paste(chunk_im, offset) - contains_chunk_luids.append(self.get_chunk_luid(chunky_x, chunky_z)) - except: - print(self.min_indent+"Could not finish "+participle+" in check_decachunk_containing_chunk:") - view_traceback() + decachunk_folder_path = self.get_decachunk_folder_path_from_decachunk(decachunky_x, decachunky_z) + if not os.path.isdir(decachunk_folder_path): + os.makedirs(decachunk_folder_path) + print(min_indent+"Made folder '"+decachunk_folder_path+"'") else: - preview_strings[chunky_offset_z] += "0" - chunky_offset_z = z_chunky_count - 1 - try: - print(self.min_indent+"Usable chunk images mask (height:"+str(z_chunky_count)+"):") - while chunky_offset_z>=0: - if preview_strings[chunky_offset_z] is None: - preview_strings[chunky_offset_z] = "" - print(self.min_indent+" "+str(chunky_offset_z)+":"+preview_strings[chunky_offset_z]) - chunky_offset_z -= 1 - except: - print(self.min_indent+"Could not finish showing mask (this should never happen)") - print(self.min_indent+" z_chunky_count:"+str(z_chunky_count)) - print(self.min_indent+" len(preview_strings):"+str(len(preview_strings))) - print(self.min_indent+" chunky_min_x:"+str(chunky_min_x)) - print(self.min_indent+" chunky_max_x:"+str(chunky_max_x)) - print(self.min_indent+" chunky_min_z:"+str(chunky_min_z)) - print(self.min_indent+" chunky_max_z:"+str(chunky_max_z)) - view_traceback() - print("") - decachunk_folder_path = self.get_decachunk_folder_path_from_decachunk(decachunky_x, decachunky_z) - if not os.path.isdir(decachunk_folder_path): - os.makedirs(decachunk_folder_path) - print(self.min_indent+"Made folder '"+decachunk_folder_path+"'") - else: - print(self.min_indent+"Found folder '"+decachunk_folder_path+"'") - print(self.min_indent+"Saving '"+decachunk_image_path+"'") - im.save(decachunk_image_path) - decachunk_luid = self.get_decachunk_luid_from_decachunk(decachunky_x, decachunky_z) - self.prepare_decachunk_meta_from_decachunk(decachunky_x, decachunky_z) - this_second = int(time.time()) - #if int(self.decachunks[decachunk_luid].metadata["last_saved_utc_second"]) != this_second: - self.decachunks[decachunk_luid].metadata["last_saved_utc_second"] = this_second # time.time() returns float even if OS doesn't give a time in increments smaller than seconds - if len(contains_chunk_luids)>0: - self.decachunks[decachunk_luid].metadata["contains_chunk_luids"] = ','.join(contains_chunk_luids) + print(min_indent+"Found folder '"+decachunk_folder_path+"'") + print(min_indent+"Saving '"+decachunk_image_path+"'") + im.save(decachunk_image_path) + decachunk_luid = self.get_decachunk_luid_from_decachunk(decachunky_x, decachunky_z) + self.prepare_decachunk_meta_from_decachunk(decachunky_x, decachunky_z) + this_second = int(time.time()) + #if int(self.decachunks[decachunk_luid].metadata["last_saved_utc_second"]) != this_second: + self.decachunks[decachunk_luid].metadata["last_saved_utc_second"] = this_second # time.time() returns float even if OS doesn't give a time in increments smaller than seconds + if len(contains_chunk_luids)>0: + self.decachunks[decachunk_luid].metadata["contains_chunk_luids"] = ','.join(contains_chunk_luids) + else: + self.decachunks[decachunk_luid].metadata["contains_chunk_luids"] = None + self.decachunks[decachunk_luid].save_yaml(decachunk_yaml_path) else: - self.decachunks[decachunk_luid].metadata["contains_chunk_luids"] = None - self.decachunks[decachunk_luid].save_yaml(decachunk_yaml_path) - else: - print(self.min_indent+"Not rendering decachunk "+str((decachunky_x,decachunky_z))+" yet since contains queued chunk "+str(queued_chunk_coords)) - print(self.min_indent+" (index:["+str(queued_index)+"]; len:"+str(len(self.todo_positions))+") .") + if is_any_part_queued: + print(min_indent+"Not rendering decachunk "+str((decachunky_x,decachunky_z))+" yet since contains queued chunk {found_index:["+str(queued_index)+"]; current_index:["+str(self.todo_index)+"]; len(todo_positions):"+str(len(self.todo_positions))+"; chunky_position:"+str(queued_chunk_coords)+"}") + else: + print(min_indent+"Not rendering decachunk "+str((decachunky_x,decachunky_z))+" yet since unfinished chunks (world border not between empty and closed area) such as empty chunk "+str(unfinished_chunky_coord)) + print(min_indent+" (index:["+str(queued_index)+"]; len:"+str(len(self.todo_positions))+") .") + except: + print(min_indent+"Could not finish check_decachunk_containing_chunk:") + view_traceback(min_indent) def get_chunk_folder_path(self, chunky_x, chunky_z): result = None @@ -900,7 +919,7 @@ class MTChunks: def remove_chunk(self, chunky_x, chunky_z): result = False - chunk_luid = get_chunk_luid(chunky_x, chunky_z) + chunk_luid = self.get_chunk_luid(chunky_x, chunky_z) out_path = self.get_chunk_genresult_tmp_path(chunky_x, chunky_z) tmp_png_path = self.get_chunk_image_path(chunky_x, chunky_z) yml_path = self.get_chunk_yaml_path(chunky_x, chunky_z) @@ -1109,7 +1128,7 @@ class MTChunks: self.prepare_chunk_meta(chunky_x, chunky_z) self.create_chunk_folder(chunky_x, chunky_z) self.chunks[chunk_luid].save_yaml(chunk_yaml_path) - print(self.min_indent+"(saved yaml to '"+chunk_yaml_path+"')") + print(min_indent+"(saved yaml to '"+chunk_yaml_path+"')") def check_players(self): print("PROCESSING PLAYERS") @@ -1405,7 +1424,7 @@ class MTChunks: #check_decachunk_containing_chunk AFTER incrementing todo_index so that self being queued doesn't prevent decachunk render: self.check_decachunk_containing_chunk(chunky_x, chunky_z) - if self.todo_index>=len(self.todo_positions): # check again since may have branched above, making this untrue + if self.todo_index>=len(self.todo_positions): # check again since may have branched above, making this untrue self.save_mapvars_if_changed() self.todo_index = -1 self.todo_positions = list() @@ -1519,7 +1538,7 @@ class MTChunks: results = tmp return results - def get_outline_coords_list(x_int, y_int, restrict_to_decachunk_enable=False): + def get_outline_coords_list(self, x_int, y_int, restrict_to_decachunk_enable=False): results = None if x_int is not None and y_int is not None: tmp = list() @@ -1569,6 +1588,9 @@ class MTChunks: return result def check_map_pseudorecursion_start(self): + if self.todo_positions is not None and self.todo_index>=len(self.todo_positions): + print("WARNING in check_map_pseudorecursion_start: todo index was ["+str(self.todo_index)+"] in "+str(len(self.todo_positions))+"-length list, so resetting todo_list") + self.todo_index = -1 if self.todo_index<0: print("PROCESSING MAP DATA (BRANCH PATTERN)") if os.path.isfile(self.minetestmapper_py_path) and os.path.isfile(self.colors_path): diff --git a/expertmm.py b/expertmm.py index f84806b..af32fab 100644 --- a/expertmm.py +++ b/expertmm.py @@ -1,6 +1,7 @@ import os import sys import traceback +import copy verbose_enable = False @@ -21,6 +22,7 @@ alpha_chars = alpha_upper_chars+alpha_lower_chars alnum_chars = alpha_chars+digit_chars identifier_chars = alnum_chars+"_" identifier_and_dot_chars = identifier_chars+"." +min_indent = "" class InstalledFile: source_dir_path = None @@ -111,7 +113,7 @@ def get_dict_deepcopy(old_dict): if type(old_dict) is dict: new_dict = {} for this_key in old_dict.iterkeys(): - new_dict[this_key] = old_dict[this_key] + new_dict[this_key] = copy.deepcopy(old_dict[this_key]) return new_dict def is_dict_subset(new_dict, old_dict, verbose_messages_enable, verbose_dest_description="unknown file"): @@ -173,8 +175,8 @@ def RepresentsFloat(s): def view_traceback(): ex_type, ex, tb = sys.exc_info() - print(str(ex_type)) - print(str(ex)) + print(min_indent+str(ex_type)) + print(min_indent+str(ex)) traceback.print_tb(tb) del tb