From 5897e8607c988c0dba9fba22ca68d836ff3afc7b Mon Sep 17 00:00:00 2001 From: poikilos <7557867+poikilos@users.noreply.github.com> Date: Tue, 22 Mar 2016 22:13:45 -0400 Subject: [PATCH] working on chunk generator --- generator.py | 141 ++++++++++++++++++++++++++++++------------------- singleimage.py | 1 + 2 files changed, 87 insertions(+), 55 deletions(-) diff --git a/generator.py b/generator.py index 487dabe..14c231c 100644 --- a/generator.py +++ b/generator.py @@ -850,7 +850,12 @@ class MTChunks: #print (min_indent+"generating chunky_x = " + str(min_x) + " to " + str(max_x) + " , chunky_z = " + str(min_z) + " to " + str(max_z)) geometry_value_string = str(min_x)+":"+str(min_z)+"+"+str(int(max_x)-int(min_x)+1)+"+"+str(int(max_z)-int(min_z)+1) # +1 since max-min is exclusive and width must be inclusive for minetestmapper.py cmd_suffix = "" - cmd_suffix = " > \""+genresult_path+"\"" + genresults_folder_path = os.path.join( os.path.join(os.path.dirname(os.path.abspath(__file__)), "chunkymap-genresults"), self.world_name) + if not os.path.isdir(genresults_folder_path): + os.makedirs(genresults_folder_path) + gen_error_path = os.path.join(genresults_folder_path, "singleimage"+gen_error_name_closer_string) + cmd_suffix = " 1> \""+genresult_path+"\"" + cmd_suffix += " 2> \""+gen_error_path+"\"" #self.mapper_id = "minetestmapper-region" cmd_no_out_string = python_exe_path + " \""+self.minetestmapper_py_path + "\" --region " + str(min_x) + " " + str(max_x) + " " + str(min_z) + " " + str(max_z) + " --maxheight "+str(self.mapvars["maxheight"])+" --minheight "+str(self.mapvars["minheight"])+" --pixelspernode "+str(self.mapvars["pixelspernode"])+" \""+minetestinfo.get_var("primary_world_path")+"\" \""+tmp_png_path+"\"" cmd_string = cmd_no_out_string + cmd_suffix @@ -951,35 +956,52 @@ class MTChunks: except: pass participle = "checking result" - try: - is_changed = this_chunk.set_from_genresult(genresult_path) - if is_marked_before: - participle = "checking for marks" - if (not is_empty_before) and this_chunk.metadata["is_empty"]: - print("ERROR: chunk changed from nonempty to empty (may happen if output of mapper was not recognized)") - elif this_chunk.metadata["is_empty"] and os.path.isfile(dest_png_path): - print("ERROR: chunk marked empty though has data (may happen if output of mapper was not recognized)") - this_is_worldborder_chunk = self.is_worldborder_chunk(chunky_x, chunky_z) - if ("is_worldborder" not in self.chunks[chunk_luid].metadata) or this_is_worldborder_chunk != self.chunks[chunk_luid].metadata["is_worldborder"]: - self.chunks[chunk_luid].metadata["is_worldborder"] = this_is_worldborder_chunk - is_changed = True + is_locked = False + if os.path.isfile(gen_error_path): + ins = open(gen_error_path, 'r') + line = True + while line: + line = ins.readline() + if line: + line_lower = line.lower() + if " lock " in line_lower or "/lock " in line_lower + is_locked = True + lock_line = line + result = None + break + ins.close() + if not is_locked: + try: + is_changed = this_chunk.set_from_genresult(genresult_path) + if is_marked_before: + participle = "checking for marks" + if (not is_empty_before) and this_chunk.metadata["is_empty"]: + print("ERROR: chunk changed from nonempty to empty (may happen if output of mapper was not recognized)") + elif this_chunk.metadata["is_empty"] and os.path.isfile(dest_png_path): + print("ERROR: chunk marked empty though has data (may happen if output of mapper was not recognized)") + this_is_worldborder_chunk = self.is_worldborder_chunk(chunky_x, chunky_z) + if ("is_worldborder" not in self.chunks[chunk_luid].metadata) or this_is_worldborder_chunk != self.chunks[chunk_luid].metadata["is_worldborder"]: + self.chunks[chunk_luid].metadata["is_worldborder"] = this_is_worldborder_chunk + is_changed = True - #chunk_yaml_path = self.get_chunk_yaml_path(chunky_x, chunky_z) - #self.create_chunk_folder(chunky_x, chunky_z) - #this_chunk.save_yaml(chunk_yaml_path) - #if is_changed: - participle = "accessing dict" - if not is_dict_subset(self.chunks[chunk_luid].metadata, old_meta, False): # , True, "chunk_yaml_path") - participle = "saving chunk meta" - self.save_chunk_meta(chunky_x, chunky_z) - #print(min_indent+"(saved yaml to '"+chunk_yaml_path+"')") - if not self.is_save_output_ok: - if os.path.isfile(genresult_path): - participle = "removing "+genresult_path - os.remove(genresult_path) - except: - print (min_indent+"Could not finish "+participle+" while deleting/moving output") - view_traceback() + #chunk_yaml_path = self.get_chunk_yaml_path(chunky_x, chunky_z) + #self.create_chunk_folder(chunky_x, chunky_z) + #this_chunk.save_yaml(chunk_yaml_path) + #if is_changed: + participle = "accessing dict" + if not is_dict_subset(self.chunks[chunk_luid].metadata, old_meta, False): # , True, "chunk_yaml_path") + participle = "saving chunk meta" + self.save_chunk_meta(chunky_x, chunky_z) + #print(min_indent+"(saved yaml to '"+chunk_yaml_path+"')") + if not self.is_save_output_ok: + if os.path.isfile(genresult_path): + participle = "removing "+genresult_path + os.remove(genresult_path) + except: + print (min_indent+"Could not finish "+participle+" while deleting/moving output") + view_traceback() + else: + print(min_indent+"database locked: "+lock_line) except: print(min_indent+"Could not finish deleting/moving temp files") view_traceback() @@ -1231,7 +1253,7 @@ class MTChunks: return result - #Returns: (boolean) whether the chunk image is present on dest (rendered now or earlier)--only possible if there is chunk data at the given location + #Returns: (boolean) whether the chunk image is present on dest (rendered now or earlier); else None if database is locked (then re-adds it to self.todo_positions)--only possible if there is chunk data at the given location def check_chunk(self, chunky_x, chunky_z): min_indent = " " result = [False,""] @@ -1323,8 +1345,15 @@ class MTChunks: self.rendered_count += 1 if not self.verbose_enable: print(min_indent+chunk_luid+": "+result[1]) - if (self._render_chunk(chunky_x, chunky_z)): + sub_result = self._render_chunk(chunky_x, chunky_z) + if (sub_result==True): result[0] = True + elif sub_result==None: + result[0] = None + self.todo_positions.append((chunky_x, chunky_z)) #redo this one + print("Waiting to retry...") + time.sleep(.5) + else: if self.is_chunk_rendered_on_dest(chunky_x, chunky_z): result[0] = True @@ -1370,28 +1399,30 @@ class MTChunks: prev_rendered_this_session_count = self.rendered_this_session_count is_present, reason_string = self.check_chunk(chunky_x, chunky_z) - if is_present: - self.mapvars["total_generated_count"] += 1 - if chunky_xself.mapvars["max_chunkx"]: - self.mapvars["max_chunkx"]=chunky_x - if chunky_zself.mapvars["max_chunkz"]: - self.mapvars["max_chunkz"]=chunky_z - #end while square outline (1-chunk-thick outline) generated any png files - self.save_mapvars_if_changed() - prev_len = len(self.todo_positions) - self._check_map_pseudorecursion_branchfrom(chunky_x, chunky_z) - #must check_decachunk_containing_chunk AFTER _check_map_pseudorecursion_branchfrom so check_decachunk_containing_chunk can see if there are more to do before rendering superchunk - #always check since already checks queue and doesn't render decachunk on last rendered chunk, but instead on last queued chunk in decachunk - #if self.rendered_this_session_count>prev_rendered_this_session_count or self.force_rerender_decachunks_enable: - - #Now is ok to check_decachunk_containing_chunk, since does not count current index as unfinished (allow_current_chunk_enable=False): - self.check_decachunk_containing_chunk(chunky_x, chunky_z) - if self.verbose_enable: - print(min_indent+"["+str(self.todo_index)+"] branching from "+str((chunky_x, chunky_z))+" (added "+str(len(self.todo_positions)-prev_len)+")") + if (is_present is None) or is_present: + if is_present: + self.mapvars["total_generated_count"] += 1 + if chunky_xself.mapvars["max_chunkx"]: + self.mapvars["max_chunkx"]=chunky_x + if chunky_zself.mapvars["max_chunkz"]: + self.mapvars["max_chunkz"]=chunky_z + #end while square outline (1-chunk-thick outline) generated any png files + self.save_mapvars_if_changed() + prev_len = len(self.todo_positions) + self._check_map_pseudorecursion_branchfrom(chunky_x, chunky_z) + #must check_decachunk_containing_chunk AFTER _check_map_pseudorecursion_branchfrom so check_decachunk_containing_chunk can see if there are more to do before rendering superchunk + #always check since already checks queue and doesn't render decachunk on last rendered chunk, but instead on last queued chunk in decachunk + #if self.rendered_this_session_count>prev_rendered_this_session_count or self.force_rerender_decachunks_enable: + + #Now is ok to check_decachunk_containing_chunk, since does not count current index as unfinished (allow_current_chunk_enable=False): + self.check_decachunk_containing_chunk(chunky_x, chunky_z) + if self.verbose_enable: + print(min_indent+"["+str(self.todo_index)+"] branching from "+str((chunky_x, chunky_z))+" (added "+str(len(self.todo_positions)-prev_len)+")") + #else None (database is locked) so let it be retried later else: #Now is ok to check_decachunk_containing_chunk, since does not count current index as unfinished (allow_current_chunk_enable=False): self.check_decachunk_containing_chunk(chunky_x, chunky_z) @@ -1402,8 +1433,8 @@ class MTChunks: if self.todo_index>=len(self.todo_positions): # check again since may have branched above, making this untrue self.save_mapvars_if_changed() self.todo_index = -1 - #self.todo_positions = list() # there seems to be issues where not empty due to delayed garbage collection - while len(self.todo_positions) > 0 : self.todo_positions.pop() + self.todo_positions = list() # there seems to be issues where not empty due to delayed garbage collection? + #while len(self.todo_positions) > 0 : self.todo_positions.pop() else: if self.verbose_enable: print(min_indent+"(no branches)") diff --git a/singleimage.py b/singleimage.py index d82f4e6..594dc7b 100644 --- a/singleimage.py +++ b/singleimage.py @@ -49,6 +49,7 @@ class ChunkymapOfflineRenderer: genresult_path = os.path.join(genresults_folder_path, "singleimage"+genresult_name_closer_string) gen_error_path = os.path.join(genresults_folder_path, "singleimage"+gen_error_name_closer_string) cmd_suffix = " 1> \""+genresult_path+"\"" + cmd_suffix += " 2> \""+gen_error_path+"\"" #blank since singleimage mode: #geometry_string = str(min_x)+":"+str(min_z)+"+"+str(int(max_x)-int(min_x)+1)+"+"+str(int(max_z)-int(min_z)+1) # +1 since max-min is exclusive and width must be inclusive for minetestmapper.py