import os import sys import keyword #import types import inspect import traceback module_list = list() module_list.append("os") module_list.append("sys") module_list.append("keyword") module_list.append("inspect") module_list.append("traceback") from expertmm import * #TODO: code checking such as: # * check for redefining member variable (missing "self.") [since python does not throw NameError on definition, only if used and not defined] # * check for signs in len params (such as where str(len(part1+part2)) should be str(len(part1))+part2 # * check for only variable name in quotes (maybe the programmer meant to use the value) # * check for use of os.dirname (should be os.path.dirname) def view_traceback(): ex_type, ex, tb = sys.exc_info() print(str(ex_type)) print(str(ex)) traceback.print_tb(tb) del tb class RegressionMismatch: side_a_string = None side_b_string = None endswith_enable = None startswith_enable = None def __init__(self, side_a_string, side_b_string, startswith_enable, endswith_enable): self.side_a_string = side_a_string self.side_b_string = side_b_string self.endswith_enable = endswith_enable self.startswith_enable = startswith_enable global_case_sensitive_enable = None y_enable = False print("Initializing...") regression_mismatches = list() independent_list = ["index","suffix","prefix"] independent_list.append("self") independent_list.append("int") independent_list.append("float") independent_list.append("double") independent_list.append("long") independent_list.append("bool") independent_list.append("str") independent_list.append("string") independent_list.append("strlen") independent_list.append("len") independent_list.append("os.path.join") independent_list.append("open") independent_list.append("close") independent_list.append("None") independent_list.append("null") independent_list.append("NULL") #NOTE: "for decachunk_z_name in os.listdir(decachunk_x_path):" is ok since z folders are in x folder independent_endswith_list = list() for word in independent_list: independent_endswith_list.append("_"+word) regression_mismatches.append(RegressionMismatch("_x","_z",False,True)) if y_enable: regression_mismatches.append(RegressionMismatch("_x","_y",False,True)) regression_mismatches.append(RegressionMismatch("_y","_z",False,True)) regression_mismatches.append(RegressionMismatch("_x_","_z_",False,False)) if y_enable: regression_mismatches.append(RegressionMismatch("_x_","_y_",False,False)) regression_mismatches.append(RegressionMismatch("_y_","_z_",False,False)) regression_mismatches.append(RegressionMismatch("x","z",False,True)) if y_enable: regression_mismatches.append(RegressionMismatch("x","y",False,True)) regression_mismatches.append(RegressionMismatch("y","z",False,True)) regression_mismatches.append(RegressionMismatch("x_","z_",True,False)) if y_enable: regression_mismatches.append(RegressionMismatch("x_","y_",True,False)) regression_mismatches.append(RegressionMismatch("y_","z_",True,False)) if global_case_sensitive_enable is True: regression_mismatches.append(RegressionMismatch("_X_","_Z_",False,False)) if y_enable: regression_mismatches.append(RegressionMismatch("_X_","_Y_",False,False)) regression_mismatches.append(RegressionMismatch("_Y_","_Z_",False,False)) regression_mismatches.append(RegressionMismatch("_X","_Z",False,True)) if y_enable: regression_mismatches.append(RegressionMismatch("_X","_Y",False,True)) regression_mismatches.append(RegressionMismatch("_Y","_Z",False,True)) regression_mismatches.append(RegressionMismatch("X","Z",False,True)) if y_enable: regression_mismatches.append(RegressionMismatch("X","Y",False,True)) regression_mismatches.append(RegressionMismatch("Y","Z",False,True)) regression_mismatches.append(RegressionMismatch("X_","Z_",True,False)) if y_enable: regression_mismatches.append(RegressionMismatch("X_","Y_",True,False)) regression_mismatches.append(RegressionMismatch("Y_","Z_",True,False)) def is_dependend_variable(name): result = False for regression_mismatch in regression_mismatches: if regression_mismatch.startswith_enable: if startswith(name, regression_mismatch.side_a_string): result = True break if startswith(name, regression_mismatch.side_b_string): result = True break elif regression_mismatch.endswith_enable: if endswith(name, regression_mismatch.side_a_string): result = True break if endswith(name, regression_mismatch.side_b_string): result = True break else: if regression_mismatch.side_a_string in name: result = True break elif regression_mismatch.side_a_string in name: result = True break return result #splits by any non-alphanumeric characters #print(keyword.kwlist) # DOESN'T WORK (from linuxbochs on http://stackoverflow.com/questions/6315496/display-a-list-of-user-defined-functions-in-the-python-idle-session ): #function_list = [f for f in globals().values() if type(f) == types.FunctionType] # DOESN'T WORK #def dummy(): pass #function_list = [f.__name__ for f in globals().values() if type(f) == type(dummy)] function_list = list() for module_string in module_list: exec ("function_list += inspect.getmembers("+module_string+", inspect.isroutine)") # NOTE: isfunction does NOT include c-defined functions such as those in math function_names = list() for function_tuple in function_list: function_names.append(function_tuple[0]) print(" (Ignoring known routines:") print(','.join(function_names)+")") print("") print("") def split_non_alnum(haystack, strip_enable=True, skip_keywords_enable=True): global independent_list global function_names global module_list results = list() index = 0 start_index = 0 while index <= len(haystack): if index==len(haystack) or not haystack[index].isalnum(): word = haystack[start_index:index] if (not skip_keywords_enable) or (word not in keyword.kwlist and word not in function_names and word not in independent_list and word not in module_list): if strip_enable: results.append(word.strip()) else: results.append(word) start_index = index+1 index += 1 return results issue_count = 0 file_list = list() #def is_identifier(needle, dot_continues_enable): # result = True # # for index in range(0,len(needle)): # if needle[index] not in identifier_chars: # result = False # break # return result quote_list = "'\"" def split_non_identifier(haystack, strip_enable=True, skip_keywords_enable=True, skip_independent_enable=True, dot_continues_enable=True): results = list() index = 0 start_index = 0 is_in_quote_char = None prev_char = None escaped_dq_enable = True while index <= len(haystack): if index0 and len(haystack)>=len(needle): if case_sensitive_enable: if haystack[-len(needle):] == needle: #print("haystack[-len(needle):] = "+haystack[-len(needle):]+" in "+haystack) result = True else: if haystack[-len(needle):].lower() == needle.lower(): #print("haystack[-len(needle):].lower() = "+haystack[-len(needle):].lower()+" in "+haystack) result = True return result def startswith(haystack, needle, case_sensitive_enable=False): if global_case_sensitive_enable is not None: case_sensitive_enable = global_case_sensitive_enable result = False if haystack is not None and needle is not None and len(needle)>0 and len(haystack)>=len(needle): if case_sensitive_enable: if haystack[:len(needle)] == needle: result = True else: if haystack[:len(needle)].lower() == needle.lower(): result = True return result def any_endswith(haystacks, needle, case_sensitive_enable=False): if global_case_sensitive_enable is not None: case_sensitive_enable = global_case_sensitive_enable result = False for haystack in haystacks: if endswith(haystack, needle, case_sensitive_enable): result = True break return result def endswith_any(haystack, needles, case_sensitive_enable=False): if global_case_sensitive_enable is not None: case_sensitive_enable = global_case_sensitive_enable result = False for needle in needles: if endswith(haystack, needle, case_sensitive_enable): result = True break return result def any_startswith(haystacks, needle, case_sensitive_enable=False): if global_case_sensitive_enable is not None: case_sensitive_enable = global_case_sensitive_enable result = False for haystack in haystacks: if startswith(haystack, needle, case_sensitive_enable): result = True break return result def startswith_any(haystack, needles, case_sensitive_enable=False): if global_case_sensitive_enable is not None: case_sensitive_enable = global_case_sensitive_enable result = False for needle in needles: if startswith(haystack, needle, case_sensitive_enable): result = True break return result def endswith_any(haystack, needles, case_sensitive_enable=False): if global_case_sensitive_enable is not None: case_sensitive_enable = global_case_sensitive_enable result = False for needle in needles: if endswith(haystack, needle, case_sensitive_enable): result = True break return result #only works on strings IF case_sensitive_enable, since then would do lower() on each element of haystacks def in_any_string(needle, haystacks, case_sensitive_enable=False): if global_case_sensitive_enable is not None: case_sensitive_enable = global_case_sensitive_enable result = False if needle is not None and len(needle)>0: for haystack in haystacks: if haystack is not None and len(haystack)>0: if case_sensitive_enable: if needle in haystack: result = True break else: if needle.lower() in haystack.lower(): result = True break return result min_indent = "" def increase_indent(): global min_indent min_indent+=" " def decrease_indent(): global min_indent if len(min_indent)>=2: min_indent = min_indent[:-2] def check_coord_mismatch(file_path): global file_list global issue_count global module_list print("Running check_coord_mismatch on "+file_path+"...") global function_list if (file_path not in file_list): file_list.append(file_path) line_counting_number = 1 ins = open(file_path, 'r') line = True global min_indent is_imported = False problematic_line_count = 0 inline_comment_delimiter = "#" inline_line_break = None file_path_lower = file_path.lower() if len(file_path)>3 and (file_path_lower[-3:]==".cs"): inline_comment_delimiter = "//" inline_line_break = ";" elif len(file_path)>2 and (file_path_lower[-2:]==".c"): inline_comment_delimiter = "//" inline_line_break = ";" elif len(file_path)>2 and (file_path_lower[-2:]==".h"): inline_comment_delimiter = "//" inline_line_break = ";" elif len(file_path)>4 and (file_path_lower[-4:]==".hpp"): inline_comment_delimiter = "//" inline_line_break = ";" elif len(file_path)>4 and (file_path_lower[-4:]==".cpp"): inline_comment_delimiter = "//" inline_line_break = ";" elif len(file_path)>4 and (file_path_lower[-4:]==".php"): inline_comment_delimiter = "//" inline_line_break = ";" elif len(file_path)>3 and (file_path_lower[-3:]==".py"): inline_comment_delimiter = "#" elif len(file_path)>4 and (file_path_lower[-4:]==".pyw"): inline_comment_delimiter = "#" else: answer=raw_input("what is the inline comment delimiter for the sourcecode file type of "+file_path+" [blank for "+inline_comment_delimiter+"]? ") if len(answer)>0: inline_comment_delimiter = answer dup_ignore_list = function_names+independent_list+module_list while line: line = ins.readline() prev_issue_count = issue_count if line: line_strip = line.strip() comment_index = find_unquoted_MAY_BE_COMMENTED(line_strip, inline_comment_delimiter) if comment_index>-1: line_strip = line_strip[:comment_index] sublines = list() if inline_line_break is not None: remaining_string = line_strip while True: inline_line_break_index = find_unquoted_MAY_BE_COMMENTED(remaining_string, inline_line_break) # do NOT check for INLINE COMMENT since already removed it using detected delimiter above if inline_line_break_index<0: break sublines.append(remaining_string[:inline_line_break_index]) remaining_string = remaining_string[inline_line_break_index+1:] if len(remaining_string)>0: sublines.append(remaining_string) else: sublines.append(line_strip) for subline in sublines: line_strip = subline if (len(line_strip)>0) and (line_strip[0]!="#"): ao_index = line_strip.find("=") import_string = "import " from_string = "from " import_index = -1 if line_strip[:len(import_string)]==import_string: import_index = 0 elif line_strip[:len(from_string)]==from_string: import_string = from_string import_index = line_strip.find(import_string) if import_index>=0: module_string = line_strip[import_index+len(import_string):].strip() space_index = module_string.find(" ") if space_index>-1: module_string = module_string[:space_index] if module_string not in module_list: module_list.append(module_string) try: #tmp_tuples = list() import_string = "import "+module_string #exec exec_string try_enable = False outs = open('expertmmregressiontmp.py','w') outs.write("def get_module_contents():"+"\n") outs_indent = " " outs.write(outs_indent+"results = None"+"\n") if try_enable: outs.write(outs_indent+"try:"+"\n") outs_indent = " " outs.write(outs_indent+"import inspect"+"\n") outs.write(outs_indent+import_string+"\n") exec_string = "tmp_tuples = inspect.getmembers("+module_string+", inspect.isroutine)" outs.write(outs_indent+exec_string+"\n") outs.write(outs_indent+"results = list()"+"\n") outs.write(outs_indent+"for function_tuple in tmp_tuples:"+"\n") outs.write(outs_indent+" results.append(function_tuple[0])"+"\n") outs_indent = " " if try_enable: outs.write(outs_indent+"except:"+"\n") outs_indent = " " outs.write(outs_indent+"print(\"Could not finish get_module_contents\")"+"\n") outs_indent = " " outs.write(outs_indent+"return results"+"\n") outs.write("\n") outs.close() if is_imported: reload(expertmmregressiontmp) else: import expertmmregressiontmp tmp_list = expertmmregressiontmp.get_module_contents() os.remove("expertmmregressiontmp.py") new_list = None if tmp_list is not None: new_list = list() for routine_string in tmp_list: if routine_string not in function_list: new_list.append(routine_string) if new_list is not None: function_list += new_list print("Found "+str(len(new_list))+" new method(s) from '"+module_string+"' to ignore: "+','.join(new_list)) if len(new_list)>0: dup_ignore_list = function_names+independent_list+module_list else: print("unable to import module named '"+module_string+"', so some routines may not be successfully ignored:") del expertmmregressiontmp except: print("Could not finish importing module named '"+module_string+"', so some routines may not be successfully ignored:") view_traceback() if ao_index<0: ao_index = line_strip.find(">") if ao_index<0: ao_index = line_strip.find("<") if ao_index<0: ao_index = line_strip.find(" in ") if ao_index>0: # intentionally >0 instead of = increase_indent() names_string = line_strip[:ao_index].strip() values_string = line_strip[ao_index+1:].strip() name_list = split_non_identifier(names_string) value_list = split_non_identifier(values_string) message = None message_prefix = " WARNING: " this_start_index = 0 names_string = line_strip[:ao_index].strip() values_string = line_strip[ao_index+1:].strip() name_list = split_non_identifier(names_string) value_list = split_non_identifier(values_string) #while True: #this_chunk_len = get_operation_chunk_len(line_strip,this_start_index,1,line_counting_number) #if this_chunk_len<1: #break #partial_string = line_strip[this_start_index:this_start_index+this_chunk_len] #this_start_index += this_chunk_len duplicate_index = find_dup(name_list,ignore_numbers_enable=True) if duplicate_index>-1: if name_list[duplicate_index] not in dup_ignore_list: if is_dependend_variable(name_list[duplicate_index]): name_index = line.find(name_list[duplicate_index]) message_prefix = " WARNING: " issue_count += 1 message = (file_path+" ("+str(line_counting_number)+","+str(name_index)+")"+message_prefix+"name '"+name_list[duplicate_index]+"' is used twice (perhaps other coord should have been used)") print(message) break duplicate_index = find_dup(value_list) if duplicate_index>-1: if value_list[duplicate_index] not in dup_ignore_list: if is_dependend_variable(value_list[duplicate_index]): name_index = line.find(value_list[duplicate_index]) message_prefix = " WARNING: " issue_count += 1 message = (file_path+" ("+str(line_counting_number)+","+str(name_index)+")"+message_prefix+"value '"+value_list[duplicate_index]+"' is used twice (perhaps other coord should have been used)") print(message) break message = None for regression_mismatch in regression_mismatches: both_present = False only_mismatched_coord_is_present_string = " ERROR (only has mismatch): " message_prefix = " WARNING: " if regression_mismatch.startswith_enable: if (any_startswith(name_list,regression_mismatch.side_a_string) and not any_startswith(name_list,regression_mismatch.side_b_string)) and any_startswith(value_list,regression_mismatch.side_b_string): name_index = line.find(regression_mismatch.side_a_string) + 1 if not both_present: both_present = any_startswith(value_list,regression_mismatch.side_a_string) message_prefix = only_mismatched_coord_is_present_string #if message is None: message = (file_path+" ("+str(line_counting_number)+","+str(name_index)+")"+message_prefix+"name starts with "+regression_mismatch.side_a_string+", but "+regression_mismatch.side_b_string+" on right in check/assignment") print(message) issue_count += 1 break elif (any_startswith(name_list,regression_mismatch.side_b_string) and not any_startswith(name_list,regression_mismatch.side_a_string)) and any_startswith(value_list,regression_mismatch.side_a_string): name_index = line.find(regression_mismatch.side_b_string) + 1 if not both_present: both_present = any_startswith(value_list,regression_mismatch.side_b_string) message_prefix = only_mismatched_coord_is_present_string #if message is None: message = (file_path+" ("+str(line_counting_number)+","+str(name_index)+")"+message_prefix+"name starts with "+regression_mismatch.side_b_string+", but "+regression_mismatch.side_a_string+" on right in check/assignment") print(message) issue_count += 1 break elif regression_mismatch.endswith_enable: if (any_endswith(name_list,regression_mismatch.side_a_string) and not any_endswith(name_list,regression_mismatch.side_b_string)) and any_endswith(value_list,regression_mismatch.side_b_string): name_index = line.find(regression_mismatch.side_a_string) + 1 if not both_present: both_present = any_endswith(value_list,regression_mismatch.side_a_string) message_prefix = only_mismatched_coord_is_present_string #if message is None: message = (file_path+" ("+str(line_counting_number)+","+str(name_index)+")"+message_prefix+"name ends with "+regression_mismatch.side_a_string+", but "+regression_mismatch.side_b_string+" on right in check/assignment") print(message) issue_count += 1 break elif (any_endswith(name_list,regression_mismatch.side_b_string) and not any_endswith(name_list,regression_mismatch.side_a_string)) and any_endswith(value_list,regression_mismatch.side_a_string): name_index = line.find(regression_mismatch.side_b_string) + 1 if not both_present: both_present = any_endswith(value_list,regression_mismatch.side_b_string) message_prefix = only_mismatched_coord_is_present_string if message is None: message = (file_path+" ("+str(line_counting_number)+","+str(name_index)+")"+message_prefix+"name ends with "+regression_mismatch.side_b_string+", but "+regression_mismatch.side_a_string+" on right in check/assignment") print(message) issue_count += 1 break else: if (in_any_string(regression_mismatch.side_a_string, name_list) and not in_any_string(regression_mismatch.side_b_string, name_list)) and in_any_string(regression_mismatch.side_b_string, value_list): value_index = line.find(regression_mismatch.side_a_string) + 1 if not both_present: both_present = in_any_string(regression_mismatch.side_a_string, value_list) message_prefix = only_mismatched_coord_is_present_string #if message is None: message = (file_path+" ("+str(line_counting_number)+","+str(value_index)+")"+message_prefix+"name contains "+regression_mismatch.side_a_string+", but "+regression_mismatch.side_b_string+" on right in check/assignment") print(message) issue_count += 1 break elif (in_any_string(regression_mismatch.side_b_string, name_list) and not in_any_string(regression_mismatch.side_a_string, name_list)) and in_any_string(regression_mismatch.side_a_string, value_list): value_index = line.find(regression_mismatch.side_b_string) + 1 if not both_present: both_present = in_any_string(regression_mismatch.side_b_string, value_list) message_prefix = only_mismatched_coord_is_present_string #if message is None: message = (file_path+" ("+str(line_counting_number)+","+str(value_index)+")"+message_prefix+"name contains "+regression_mismatch.side_b_string+", but "+regression_mismatch.side_a_string+" on right side of check/assignment") print(message) issue_count += 1 break if issue_count>prev_issue_count: print("line "+str(line_counting_number)+": "+line.strip()) if verbose_enable: print(" ao_index:"+str(ao_index)) print(" names: "+','.join(name_list)) print(" values: "+','.join(value_list)) print("") problematic_line_count += 1 line_counting_number += 1 ins.close() independent_list.append("decachunk_x_path") independent_list.append("chunk_assoc") independent_list.append("chunk_luid") independent_list.append("file_name") independent_list.append("chunkymap_view_zoom_multiplier") independent_list.append("\"0\"") independent_list.append("temp") independent_list.append("haystack") independent_list.append("needle") independent_list.append("line_strip") independent_list.append("self.chunks") independent_list.append("self.mapvars") independent_list.append(".metadata") independent_list.append("\" \\\"\"") independent_list.append("\" \"") independent_list.append("player_position_tuple") print(" (Ignoring the following independent variables:") print(','.join(independent_list)+")") #verbose_enable = True check_coord_mismatch("chunkymap-regen.py") check_coord_mismatch(os.path.join("web","chunkymap.php")) print("Found "+str(issue_count)+" issue(s) in "+str(len(file_list))+" file(s)") if (issue_count>0): print("Please run again after these issues are fixed to check for more on same lines.") raw_input("Press enter to exit...")