#!/usr/bin/env python3"""This is a middle-processor for MicroPython source files. It takes the outputof the C preprocessor, has the option to change it, then feeds this into theC compiler.It currently has the ability to reorder static hash tables so they are actuallyhashed, resulting in faster lookup times at runtime.To use, configure the Python variables below, and add the following line to theMakefile:CFLAGS += -no-integrated-cpp -B$(shell pwd)/../tools"""import sysimport osimport re################################################################################# these are the configuration variables# TODO somehow make them externally configurable# this is the path to the true C compilercc1_path = '/usr/lib/gcc/x86_64-unknown-linux-gnu/5.3.0/cc1'#cc1_path = '/usr/lib/gcc/arm-none-eabi/5.3.0/cc1'# this must be the same as MICROPY_QSTR_BYTES_IN_HASHbytes_in_qstr_hash = 2# this must be 1 or more (can be a decimal)# larger uses more code size but yields faster lookupstable_size_mult = 1# these control output during processingprint_stats = Trueprint_debug = False# end configuration variables################################################################################# precompile regexsre_preproc_line = re.compile(r'# [0-9]+ ')re_map_entry = re.compile(r'\{.+?\(MP_QSTR_([A-Za-z0-9_]+)\).+\},')re_mp_obj_dict_t = re.compile(r'(?P<head>(static )?const mp_obj_dict_t (?P<id>[a-z0-9_]+) = \{ \.base = \{&mp_type_dict\}, \.map = \{ \.all_keys_are_qstrs = 1, \.is_fixed = 1, \.is_ordered = )1(?P<tail>, \.used = .+ };)$')re_mp_map_t = re.compile(r'(?P<head>(static )?const mp_map_t (?P<id>[a-z0-9_]+) = \{ \.all_keys_are_qstrs = 1, \.is_fixed = 1, \.is_ordered = )1(?P<tail>, \.used = .+ };)$')re_mp_rom_map_elem_t = re.compile(r'static const mp_rom_map_elem_t [a-z_0-9]+\[\] = {$')# this must match the equivalent function in qstr.cdef compute_hash(qstr):hash = 5381for char in qstr:hash = (hash * 33) ^ ord(char)# Make sure that valid hash is never zero, zero means "hash not computed"return (hash & ((1 << (8 * bytes_in_qstr_hash)) - 1)) or 1# this algo must match the equivalent in map.cdef hash_insert(map, key, value):hash = compute_hash(key)pos = hash % len(map)start_pos = posif print_debug:print(' insert %s: start at %u/%u -- ' % (key, pos, len(map)), end='')while True:if map[pos] is None:# found empty slot, so key is not in tableif print_debug:print('put at %u' % pos)map[pos] = (key, value)returnelse:# not yet found, keep searchingif map[pos][0] == key:raise AssertionError("duplicate key '%s'" % (key,))pos = (pos + 1) % len(map)assert pos != start_posdef hash_find(map, key):hash = compute_hash(key)pos = hash % len(map)start_pos = posattempts = 0while True:attempts += 1if map[pos] is None:return attempts, Noneelif map[pos][0] == key:return attempts, map[pos][1]else:pos = (pos + 1) % len(map)if pos == start_pos:return attempts, Nonedef process_map_table(file, line, output):output.append(line)# consume all lines that are entries of the table and concat them# (we do it this way because there can be multiple entries on one line)table_contents = []while True:line = file.readline()if len(line) == 0:print('unexpected end of input')sys.exit(1)line = line.strip()if len(line) == 0:# empty linecontinueif re_preproc_line.match(line):# preprocessor line number commentcontinueif line == '};':# end of table (we assume it appears on a single line)breaktable_contents.append(line)# make combined string of entriesentries_str = ''.join(table_contents)# split into individual entriesentries = []while entries_str:# look for single entry, by matching nested bracesmatch = Noneif entries_str[0] == '{':nested_braces = 0for i in range(len(entries_str)):if entries_str[i] == '{':nested_braces += 1elif entries_str[i] == '}':nested_braces -= 1if nested_braces == 0:match = re_map_entry.match(entries_str[:i + 2])breakif not match:print('unknown line in table:', entries_str)sys.exit(1)# extract single entryline = match.group(0)qstr = match.group(1)entries_str = entries_str[len(line):].lstrip()# add the qstr and the whole line to list of all entriesentries.append((qstr, line))# sort entries so hash table construction is deterministicentries.sort()# create hash tablemap = [None] * int(len(entries) * table_size_mult)for qstr, line in entries:# We assume that qstr does not have any escape sequences in it.# This is reasonably safe, since keys in a module or class dict# should be standard identifiers.# TODO verify this and raise an error if escape sequence foundhash_insert(map, qstr, line)# compute statisticstotal_attempts = 0for qstr, _ in entries:attempts, line = hash_find(map, qstr)assert line is not Noneif print_debug:print(' %s lookup took %u attempts' % (qstr, attempts))total_attempts += attemptsif len(entries):stats = len(map), len(entries) / len(map), total_attempts / len(entries)else:stats = 0, 0, 0if print_debug:print(' table stats: size=%d, load=%.2f, avg_lookups=%.1f' % stats)# output hash tablefor row in map:if row is None:output.append('{ 0, 0 },\n')else:output.append(row[1] + '\n')output.append('};\n')# skip to next non-blank linewhile True:line = file.readline()if len(line) == 0:print('unexpected end of input')sys.exit(1)line = line.strip()if len(line) == 0:continuebreak# transform the is_ordered param from 1 to 0match = re_mp_obj_dict_t.match(line)if match is None:match = re_mp_map_t.match(line)if match is None:print('expecting mp_obj_dict_t or mp_map_t definition')print(output[0])print(line)sys.exit(1)line = match.group('head') + '0' + match.group('tail') + '\n'output.append(line)return (match.group('id'),) + statsdef process_file(filename):output = []file_changed = Falsewith open(filename, 'rt') as f:while True:line = f.readline()if not line:breakif re_mp_rom_map_elem_t.match(line):file_changed = Truestats = process_map_table(f, line, output)if print_stats:print(' [%s: size=%d, load=%.2f, avg_lookups=%.1f]' % stats)else:output.append(line)if file_changed:if print_debug:print(' modifying static maps in', output[0].strip())with open(filename, 'wt') as f:for line in output:f.write(line)def main():# run actual C compiler# need to quote args that have special characters in themdef quote(s):if s.find('<') != -1 or s.find('>') != -1:return "'" + s + "'"else:return sret = os.system(cc1_path + ' ' + ' '.join(quote(s) for s in sys.argv[1:]))if ret != 0:ret = (ret & 0x7f) or 127 # make it in range 0-127, but non-zerosys.exit(ret)if sys.argv[1] == '-E':# CPP has been run, now do our processing stagefor i, arg in enumerate(sys.argv):if arg == '-o':return process_file(sys.argv[i + 1])print('%s: could not find "-o" option' % (sys.argv[0],))sys.exit(1)elif sys.argv[1] == '-fpreprocessed':# compiler has been run, nothing more to doreturnelse:# unknown processing stageprint('%s: unknown first option "%s"' % (sys.argv[0], sys.argv[1]))sys.exit(1)if __name__ == '__main__':main()
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。