[Python-checkins] gh-90997: bpo-46841: Disassembly of quickened code (GH-32099)

Tue Apr 19 04:45:24 EDT 2022

https://github.com/python/cpython/commit/e590379197f065f52c8140c0edd7a59360216531
commit: e590379197f065f52c8140c0edd7a59360216531
branch: main
author: penguin_wwy <940375606 at qq.com>
committer: markshannon <mark at hotpy.org>
date: 2022年04月19日T09:45:08+01:00
summary:
gh-90997: bpo-46841: Disassembly of quickened code (GH-32099)
files:
A Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst
M Lib/dis.py
M Lib/test/test__opcode.py
M Lib/test/test_dis.py

diff --git a/Lib/dis.py b/Lib/dis.py
index f7b38a82ab13e..205e9d8d193f3 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -7,7 +7,7 @@
 
 from opcode import *
 from opcode import __all__ as _opcodes_all
-from opcode import _nb_ops
+from opcode import _nb_ops, _inline_cache_entries, _specializations, _specialized_instructions
 
 __all__ = ["code_info", "dis", "disassemble", "distb", "disco",
 "findlinestarts", "findlabels", "show_code",
@@ -34,6 +34,18 @@
 
 CACHE = opmap["CACHE"]
 
+_all_opname = list(opname)
+_all_opmap = dict(opmap)
+_empty_slot = [slot for slot, name in enumerate(_all_opname) if name.startswith("<")]
+for spec_op, specialized in zip(_empty_slot, _specialized_instructions):
+ # fill opname and opmap
+ _all_opname[spec_op] = specialized
+ _all_opmap[specialized] = spec_op
+
+deoptmap = {
+ specialized: base for base, family in _specializations.items() for specialized in family
+}
+
 def _try_compile(source, name):
 """Attempts to compile the given source, first as an expression and
 then as a statement if the first approach fails.
@@ -47,7 +59,7 @@ def _try_compile(source, name):
 c = compile(source, name, 'exec')
 return c
 
-def dis(x=None, *, file=None, depth=None, show_caches=False):
+def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False):
 """Disassemble classes, methods, functions, and other compiled objects.
 
 With no argument, disassemble the last traceback.
@@ -57,7 +69,7 @@ def dis(x=None, *, file=None, depth=None, show_caches=False):
 in a special attribute.
 """
 if x is None:
- distb(file=file, show_caches=show_caches)
+ distb(file=file, show_caches=show_caches, adaptive=adaptive)
 return
 # Extract functions from methods.
 if hasattr(x, '__func__'):
@@ -78,21 +90,21 @@ def dis(x=None, *, file=None, depth=None, show_caches=False):
 if isinstance(x1, _have_code):
 print("Disassembly of %s:" % name, file=file)
 try:
- dis(x1, file=file, depth=depth, show_caches=show_caches)
+ dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
 except TypeError as msg:
 print("Sorry:", msg, file=file)
 print(file=file)
 elif hasattr(x, 'co_code'): # Code object
- _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches)
+ _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
 elif isinstance(x, (bytes, bytearray)): # Raw bytecode
 _disassemble_bytes(x, file=file, show_caches=show_caches)
 elif isinstance(x, str): # Source code
- _disassemble_str(x, file=file, depth=depth, show_caches=show_caches)
+ _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
 else:
 raise TypeError("don't know how to disassemble %s objects" %
 type(x).__name__)
 
-def distb(tb=None, *, file=None, show_caches=False):
+def distb(tb=None, *, file=None, show_caches=False, adaptive=False):
 """Disassemble a traceback (default: last traceback)."""
 if tb is None:
 try:
@@ -100,7 +112,7 @@ def distb(tb=None, *, file=None, show_caches=False):
 except AttributeError:
 raise RuntimeError("no last traceback to disassemble") from None
 while tb.tb_next: tb = tb.tb_next
- disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches)
+ disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive)
 
 # The inspect module interrogates this dictionary to build its
 # list of CO_* constants. It is also used by pretty_flags to
@@ -162,6 +174,13 @@ def _get_code_object(x):
 raise TypeError("don't know how to disassemble %s objects" %
 type(x).__name__)
 
+def _deoptop(op):
+ name = _all_opname[op]
+ return _all_opmap[deoptmap[name]] if name in deoptmap else op
+
+def _get_code_array(co, adaptive):
+ return co._co_code_adaptive if adaptive else co.co_code
+
 def code_info(x):
 """Formatted details of methods, functions, or code."""
 return _format_code_info(_get_code_object(x))
@@ -302,7 +321,7 @@ def _disassemble(self, lineno_width=3, mark_as_current=False, offset_width=4):
 return ' '.join(fields).rstrip()
 
 
-def get_instructions(x, *, first_line=None, show_caches=False):
+def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False):
 """Iterator for the opcodes in methods, functions or code
 
 Generates a series of Instruction named tuples giving the details of
@@ -319,7 +338,7 @@ def get_instructions(x, *, first_line=None, show_caches=False):
 line_offset = first_line - co.co_firstlineno
 else:
 line_offset = 0
- return _get_instructions_bytes(co.co_code,
+ return _get_instructions_bytes(_get_code_array(co, adaptive),
 co._varname_from_oparg,
 co.co_names, co.co_consts,
 linestarts, line_offset,
@@ -415,8 +434,13 @@ def _get_instructions_bytes(code, varname_from_oparg=None,
 for i in range(start, end):
 labels.add(target)
 starts_line = None
+ cache_counter = 0
 for offset, op, arg in _unpack_opargs(code):
- if not show_caches and op == CACHE:
+ if cache_counter > 0:
+ if show_caches:
+ yield Instruction("CACHE", 0, None, None, '',
+ offset, None, False, None)
+ cache_counter -= 1
 continue
 if linestarts is not None:
 starts_line = linestarts.get(offset, None)
@@ -426,61 +450,63 @@ def _get_instructions_bytes(code, varname_from_oparg=None,
 argval = None
 argrepr = ''
 positions = Positions(*next(co_positions, ()))
+ deop = _deoptop(op)
+ cache_counter = _inline_cache_entries[deop]
 if arg is not None:
 # Set argval to the dereferenced value of the argument when
 # available, and argrepr to the string representation of argval.
 # _disassemble_bytes needs the string repr of the
 # raw name index for LOAD_GLOBAL, LOAD_CONST, etc.
 argval = arg
- if op in hasconst:
- argval, argrepr = _get_const_info(op, arg, co_consts)
- elif op in hasname:
- if op == LOAD_GLOBAL:
+ if deop in hasconst:
+ argval, argrepr = _get_const_info(deop, arg, co_consts)
+ elif deop in hasname:
+ if deop == LOAD_GLOBAL:
 argval, argrepr = _get_name_info(arg//2, get_name)
 if (arg & 1) and argrepr:
 argrepr = "NULL + " + argrepr
 else:
 argval, argrepr = _get_name_info(arg, get_name)
- elif op in hasjabs:
+ elif deop in hasjabs:
 argval = arg*2
 argrepr = "to " + repr(argval)
- elif op in hasjrel:
- signed_arg = -arg if _is_backward_jump(op) else arg
+ elif deop in hasjrel:
+ signed_arg = -arg if _is_backward_jump(deop) else arg
 argval = offset + 2 + signed_arg*2
 argrepr = "to " + repr(argval)
- elif op in haslocal or op in hasfree:
+ elif deop in haslocal or deop in hasfree:
 argval, argrepr = _get_name_info(arg, varname_from_oparg)
- elif op in hascompare:
+ elif deop in hascompare:
 argval = cmp_op[arg]
 argrepr = argval
- elif op == FORMAT_VALUE:
+ elif deop == FORMAT_VALUE:
 argval, argrepr = FORMAT_VALUE_CONVERTERS[arg & 0x3]
 argval = (argval, bool(arg & 0x4))
 if argval[1]:
 if argrepr:
 argrepr += ', '
 argrepr += 'with format'
- elif op == MAKE_FUNCTION:
+ elif deop == MAKE_FUNCTION:
 argrepr = ', '.join(s for i, s in enumerate(MAKE_FUNCTION_FLAGS)
 if arg & (1<<i))
- elif op == BINARY_OP:
+ elif deop == BINARY_OP:
 _, argrepr = _nb_ops[arg]
- yield Instruction(opname[op], op,
+ yield Instruction(_all_opname[op], op,
 arg, argval, argrepr,
 offset, starts_line, is_jump_target, positions)
 
-def disassemble(co, lasti=-1, *, file=None, show_caches=False):
+def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False):
 """Disassemble a code object."""
 linestarts = dict(findlinestarts(co))
 exception_entries = parse_exception_table(co)
- _disassemble_bytes(co.co_code, lasti,
- co._varname_from_oparg,
+ _disassemble_bytes(_get_code_array(co, adaptive),
+ lasti, co._varname_from_oparg,
 co.co_names, co.co_consts, linestarts, file=file,
 exception_entries=exception_entries,
 co_positions=co.co_positions(), show_caches=show_caches)
 
-def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False):
- disassemble(co, file=file, show_caches=show_caches)
+def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False):
+ disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive)
 if depth is None or depth > 0:
 if depth is not None:
 depth = depth - 1
@@ -489,7 +515,7 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False):
 print(file=file)
 print("Disassembly of %r:" % (x,), file=file)
 _disassemble_recursive(
- x, file=file, depth=depth, show_caches=show_caches
+ x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive
 )
 
 def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None,
@@ -548,7 +574,7 @@ def _unpack_opargs(code):
 extended_arg = 0
 for i in range(0, len(code), 2):
 op = code[i]
- if op >= HAVE_ARGUMENT:
+ if _deoptop(op) >= HAVE_ARGUMENT:
 arg = code[i+1] | extended_arg
 extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
 # The oparg is stored as a signed integer
@@ -641,7 +667,7 @@ class Bytecode:
 
 Iterating over this yields the bytecode operations as Instruction instances.
 """
- def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False):
+ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False):
 self.codeobj = co = _get_code_object(x)
 if first_line is None:
 self.first_line = co.co_firstlineno
@@ -654,10 +680,11 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False
 self.current_offset = current_offset
 self.exception_entries = parse_exception_table(co)
 self.show_caches = show_caches
+ self.adaptive = adaptive
 
 def __iter__(self):
 co = self.codeobj
- return _get_instructions_bytes(co.co_code,
+ return _get_instructions_bytes(_get_code_array(co, self.adaptive),
 co._varname_from_oparg,
 co.co_names, co.co_consts,
 self._linestarts,
@@ -671,12 +698,12 @@ def __repr__(self):
 self._original_object)
 
 @classmethod
- def from_traceback(cls, tb, *, show_caches=False):
+ def from_traceback(cls, tb, *, show_caches=False, adaptive=False):
 """ Construct a Bytecode from the given traceback """
 while tb.tb_next:
 tb = tb.tb_next
 return cls(
- tb.tb_frame.f_code, current_offset=tb.tb_lasti, show_caches=show_caches
+ tb.tb_frame.f_code, current_offset=tb.tb_lasti, show_caches=show_caches, adaptive=adaptive
 )
 
 def info(self):
@@ -691,7 +718,7 @@ def dis(self):
 else:
 offset = -1
 with io.StringIO() as output:
- _disassemble_bytes(co.co_code,
+ _disassemble_bytes(_get_code_array(co, self.adaptive),
 varname_from_oparg=co._varname_from_oparg,
 names=co.co_names, co_consts=co.co_consts,
 linestarts=self._linestarts,
diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py
index 7c1c0cfdb069b..2a4c0d2eeb656 100644
--- a/Lib/test/test__opcode.py
+++ b/Lib/test/test__opcode.py
@@ -18,7 +18,7 @@ def test_stack_effect(self):
 self.assertRaises(ValueError, stack_effect, dis.opmap['BUILD_SLICE'])
 self.assertRaises(ValueError, stack_effect, dis.opmap['POP_TOP'], 0)
 # All defined opcodes
- for name, code in dis.opmap.items():
+ for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
 with self.subTest(opname=name):
 if code < dis.HAVE_ARGUMENT:
 stack_effect(code)
@@ -47,7 +47,7 @@ def test_stack_effect_jump(self):
 self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0)
 # All defined opcodes
 has_jump = dis.hasjabs + dis.hasjrel
- for name, code in dis.opmap.items():
+ for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
 with self.subTest(opname=name):
 if code < dis.HAVE_ARGUMENT:
 common = stack_effect(code)
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index fbc34a5dbe4ef..f560a5556c8b0 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -7,7 +7,7 @@
 import sys
 import types
 import unittest
-from test.support import captured_stdout, requires_debug_ranges
+from test.support import captured_stdout, requires_debug_ranges, cpython_only
 from test.support.bytecode_helper import BytecodeTestCase
 
 import opcode
@@ -583,6 +583,58 @@ def foo(x):
 _h.__code__.co_firstlineno + 3,
 )
 
+def load_test(x, y=0):
+ a, b = x, y
+ return a, b
+
+dis_load_test_quickened_code = """\
+%3d 0 RESUME_QUICK 0
+
+%3d 2 LOAD_FAST__LOAD_FAST 0 (x)
+ 4 LOAD_FAST 1 (y)
+ 6 STORE_FAST__STORE_FAST 3 (b)
+ 8 STORE_FAST__LOAD_FAST 2 (a)
+
+%3d 10 LOAD_FAST__LOAD_FAST 2 (a)
+ 12 LOAD_FAST 3 (b)
+ 14 BUILD_TUPLE 2
+ 16 RETURN_VALUE
+""" % (load_test.__code__.co_firstlineno,
+ load_test.__code__.co_firstlineno + 1,
+ load_test.__code__.co_firstlineno + 2)
+
+def loop_test():
+ for i in [1, 2, 3] * 3:
+ load_test(i)
+
+dis_loop_test_quickened_code = """\
+%3d 0 RESUME_QUICK 0
+
+%3d 2 BUILD_LIST 0
+ 4 LOAD_CONST 1 ((1, 2, 3))
+ 6 LIST_EXTEND 1
+ 8 LOAD_CONST 2 (3)
+ 10 BINARY_OP_ADAPTIVE 5 (*)
+ 14 GET_ITER
+ 16 FOR_ITER 17 (to 52)
+ 18 STORE_FAST 0 (i)
+
+%3d 20 LOAD_GLOBAL_MODULE 1 (NULL + load_test)
+ 32 LOAD_FAST 0 (i)
+ 34 PRECALL_PYFUNC 1
+ 38 CALL_PY_WITH_DEFAULTS 1
+ 48 POP_TOP
+ 50 JUMP_BACKWARD_QUICK 18 (to 16)
+
+%3d >> 52 LOAD_CONST 0 (None)
+ 54 RETURN_VALUE
+""" % (loop_test.__code__.co_firstlineno,
+ loop_test.__code__.co_firstlineno + 1,
+ loop_test.__code__.co_firstlineno + 2,
+ loop_test.__code__.co_firstlineno + 1,)
+
+QUICKENING_WARMUP_DELAY = 8
+
 class DisTestBase(unittest.TestCase):
 "Common utilities for DisTests and TestDisTraceback"
 
@@ -860,6 +912,93 @@ def check(expected, **kwargs):
 check(dis_nested_2, depth=None)
 check(dis_nested_2)
 
+ @staticmethod
+ def code_quicken(f, times=QUICKENING_WARMUP_DELAY):
+ for _ in range(times):
+ f()
+
+ @cpython_only
+ def test_super_instructions(self):
+ self.code_quicken(lambda: load_test(0, 0))
+ got = self.get_disassembly(load_test, adaptive=True)
+ self.do_disassembly_compare(got, dis_load_test_quickened_code, True)
+
+ @cpython_only
+ def test_binary_specialize(self):
+ binary_op_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 LOAD_NAME 0 (a)
+ 4 LOAD_NAME 1 (b)
+ 6 %s
+ 10 RETURN_VALUE
+"""
+ co_int = compile('a + b', "<int>", "eval")
+ self.code_quicken(lambda: exec(co_int, {}, {'a': 1, 'b': 2}))
+ got = self.get_disassembly(co_int, adaptive=True)
+ self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_INT 0 (+)", True)
+
+ co_unicode = compile('a + b', "<unicode>", "eval")
+ self.code_quicken(lambda: exec(co_unicode, {}, {'a': 'a', 'b': 'b'}))
+ got = self.get_disassembly(co_unicode, adaptive=True)
+ self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)", True)
+
+ binary_subscr_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 LOAD_NAME 0 (a)
+ 4 LOAD_CONST 0 (0)
+ 6 %s
+ 16 RETURN_VALUE
+"""
+ co_list = compile('a[0]', "<list>", "eval")
+ self.code_quicken(lambda: exec(co_list, {}, {'a': [0]}))
+ got = self.get_disassembly(co_list, adaptive=True)
+ self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_LIST_INT", True)
+
+ co_dict = compile('a[0]', "<dict>", "eval")
+ self.code_quicken(lambda: exec(co_dict, {}, {'a': {0: '1'}}))
+ got = self.get_disassembly(co_dict, adaptive=True)
+ self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_DICT", True)
+
+ @cpython_only
+ def test_load_attr_specialize(self):
+ load_attr_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 LOAD_CONST 0 ('a')
+ 4 LOAD_ATTR_SLOT 0 (__class__)
+ 14 RETURN_VALUE
+"""
+ co = compile("'a'.__class__", "", "eval")
+ self.code_quicken(lambda: exec(co, {}, {}))
+ got = self.get_disassembly(co, adaptive=True)
+ self.do_disassembly_compare(got, load_attr_quicken, True)
+
+ @cpython_only
+ def test_call_specialize(self):
+ call_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 PUSH_NULL
+ 4 LOAD_NAME 0 (str)
+ 6 LOAD_CONST 0 (1)
+ 8 PRECALL_NO_KW_STR_1 1
+ 12 CALL_ADAPTIVE 1
+ 22 RETURN_VALUE
+"""
+ co = compile("str(1)", "", "eval")
+ self.code_quicken(lambda: exec(co, {}, {}))
+ got = self.get_disassembly(co, adaptive=True)
+ self.do_disassembly_compare(got, call_quicken, True)
+
+ @cpython_only
+ def test_loop_quicken(self):
+ # Loop can trigger a quicken where the loop is located
+ self.code_quicken(loop_test, 1)
+ got = self.get_disassembly(loop_test, adaptive=True)
+ self.do_disassembly_compare(got, dis_loop_test_quickened_code, True)
+
 
 class DisWithFileTests(DisTests):
 
diff --git a/Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst b/Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst
new file mode 100644
index 0000000000000..0e778047593a7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst
@@ -0,0 +1 @@
+Disassembly of quickened code.
