pythongh-94216: add pseudo instructions to the dis/opcodes modules (pythonGH-94241)

Author: iritkatriel

Doc/library/dis.rst

@@ -1351,13 +1351,74 @@ iterations of the loop.
 .. opcode:: HAVE_ARGUMENT
 
    This is not really an opcode.  It identifies the dividing line between
-   opcodes which don't use their argument and those that do
-   (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively).
+   opcodes in the range [0,255] which don't use their argument and those
+   that do (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively).
+
+   If your application uses pseudo instructions, use the :data:`hasarg`
+   collection instead.
 
    .. versionchanged:: 3.6
       Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT``
       ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument.
 
+   .. versionchanged:: 3.12
+      Pseudo instructions were added to the :mod:`dis` module, and for them
+      it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether
+      they use their arg.
+
+
+**Pseudo-instructions**
+
+These opcodes do not appear in python bytecode, they are used by the compiler
+but are replaced by real opcodes or removed before bytecode is generated.
+
+.. opcode:: SETUP_FINALLY (target)
+
+   Set up an exception handler for the following code block. If an exception
+   occurs, the value stack level is restored to its current state and control
+   is transferred to the exception handler at ``target``.
+
+
+.. opcode:: SETUP_CLEANUP (target)
+
+   Like ``SETUP_FINALLY``, but in case of exception also pushes the last
+   instruction (``lasti``) to the stack so that ``RERAISE`` can restore it.
+   If an exception occurs, the value stack level and the last instruction on
+   the frame are restored to their current state, and control is transferred
+   to the exception handler at ``target``.
+
+
+.. opcode:: SETUP_WITH (target)
+
+   Like ``SETUP_CLEANUP``, but in case of exception one more item is popped
+   from the stack before control is transferred to the exception handler at
+   ``target``.
+
+   This variant is used in :keyword:`with` and :keyword:`async with`
+   constructs, which push the return value of the context manager's
+   :meth:`~object.__enter__` or :meth:`~object.__aenter__` to the stack.
+
+
+.. opcode:: POP_BLOCK
+
+   Marks the end of the code block associated with the last ``SETUP_FINALLY``,
+   ``SETUP_CLEANUP`` or ``SETUP_WITH``.
+
+.. opcode:: JUMP
+.. opcode:: JUMP_NO_INTERRUPT
+.. opcode:: POP_JUMP_IF_FALSE
+.. opcode:: POP_JUMP_IF_TRUE
+.. opcode:: POP_JUMP_IF_NONE
+.. opcode:: POP_JUMP_IF_NOT_NONE
+
+   Undirected relative jump instructions which are replaced by their
+   directed (forward/backward) counterparts by the assembler.
+
+.. opcode:: LOAD_METHOD
+
+   Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode
+   with a flag set in the arg.
+
 
 .. _opcode_collections:
 
@@ -1367,6 +1428,10 @@ Opcode collections
 These collections are provided for automatic introspection of bytecode
 instructions:
 
+   .. versionchanged:: 3.12
+      The collections now contain pseudo instructions as well. These are
+      opcodes with values ``>= MIN_PSEUDO_OPCODE``.
+
 .. data:: opname
 
    Sequence of operation names, indexable using the bytecode.
@@ -1382,6 +1447,13 @@ instructions:
    Sequence of all compare operation names.
 
 
+.. data:: hasarg
+
+   Sequence of bytecodes that use their argument.
+
+    .. versionadded:: 3.12
+
+
 .. data:: hasconst
 
    Sequence of bytecodes that access a constant.
@@ -1418,3 +1490,9 @@ instructions:
 .. data:: hascompare
 
    Sequence of bytecodes of Boolean operations.
+
+.. data:: hasexc
+
+   Sequence of bytecodes that set an exception handler.
+
+    .. versionadded:: 3.12

Doc/whatsnew/3.12.rst

@@ -93,6 +93,17 @@ New Modules
 Improved Modules
 ================
 
+dis
+---
+
+* Pseudo instruction opcodes (which are used by the compiler but
+  do not appear in executable bytecode) are now exposed in the
+  :mod:`dis` module.
+  :data:`~dis.HAVE_ARGUMENT` is still relevant to real opcodes,
+  but it is not useful for pseudo instrcutions. Use the new
+  :data:`~dis.hasarg` collection instead.
+  (Contributed by Irit Katriel in :gh:`94216`.)
+
 os
 --
 

Include/internal/pycore_opcode.h

@@ -610,7 +610,7 @@ def _unpack_opargs(code):
         op = code[i]
         deop = _deoptop(op)
         caches = _inline_cache_entries[deop]
-        if deop >= HAVE_ARGUMENT:
+        if deop in hasarg:
             arg = code[i+1] | extended_arg
             extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0
             # The oparg is stored as a signed integer

Include/opcode.h

@@ -4,9 +4,9 @@
 operate on bytecodes (e.g. peephole optimizers).
 """
 
-__all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
-           "haslocal", "hascompare", "hasfree", "opname", "opmap",
-           "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"]
+__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs",
+           "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap",
+           "HAVE_ARGUMENT", "EXTENDED_ARG"]
 
 # It's a chicken-and-egg I'm afraid:
 # We're imported before _opcode's made.
@@ -23,20 +23,29 @@
 
 cmp_op = ('<', '<=', '==', '!=', '>', '>=')
 
+hasarg = []
 hasconst = []
 hasname = []
 hasjrel = []
 hasjabs = []
 haslocal = []
 hascompare = []
 hasfree = []
-hasnargs = [] # unused
+hasexc = []
+
+def is_pseudo(op):
+    return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE
+
+oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs,
+           haslocal, hascompare, hasfree, hasexc]
 
 opmap = {}
-opname = ['<%r>' % (op,) for op in range(256)]
+
+## pseudo opcodes (used in the compiler) mapped to the values
+## they can become in the actual code.
+_pseudo_ops = {}
 
 def def_op(name, op):
-    opname[op] = name
     opmap[name] = op
 
 def name_op(name, op):
@@ -51,6 +60,17 @@ def jabs_op(name, op):
     def_op(name, op)
     hasjabs.append(op)
 
+def pseudo_op(name, op, real_ops):
+    def_op(name, op)
+    _pseudo_ops[name] = real_ops
+    # add the pseudo opcode to the lists its targets are in
+    for oplist in oplists:
+        res = [opmap[rop] in oplist for rop in real_ops]
+        if any(res):
+            assert all(res)
+            oplist.append(op)
+
+
 # Instruction opcodes for compiled code
 # Blank lines correspond to available opcodes
 
@@ -105,7 +125,7 @@ def jabs_op(name, op):
 def_op('PREP_RERAISE_STAR', 88)
 def_op('POP_EXCEPT', 89)
 
-HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
+HAVE_ARGUMENT = 90             # real opcodes from here have an argument:
 
 name_op('STORE_NAME', 90)       # Index in name list
 name_op('DELETE_NAME', 91)      # ""
@@ -200,8 +220,34 @@ def jabs_op(name, op):
 jrel_op('POP_JUMP_BACKWARD_IF_FALSE', 175)
 jrel_op('POP_JUMP_BACKWARD_IF_TRUE', 176)
 
+hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT])
+
+MIN_PSEUDO_OPCODE = 256
+
+pseudo_op('SETUP_FINALLY', 256, ['NOP'])
+hasexc.append(256)
+pseudo_op('SETUP_CLEANUP', 257, ['NOP'])
+hasexc.append(257)
+pseudo_op('SETUP_WITH', 258, ['NOP'])
+hasexc.append(258)
+pseudo_op('POP_BLOCK', 259, ['NOP'])
+
+pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD'])
+pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT'])
+pseudo_op('POP_JUMP_IF_FALSE', 262, ['POP_JUMP_FORWARD_IF_FALSE', 'POP_JUMP_BACKWARD_IF_FALSE'])
+pseudo_op('POP_JUMP_IF_TRUE', 263, ['POP_JUMP_FORWARD_IF_TRUE', 'POP_JUMP_BACKWARD_IF_TRUE'])
+pseudo_op('POP_JUMP_IF_NONE', 264, ['POP_JUMP_FORWARD_IF_NONE', 'POP_JUMP_BACKWARD_IF_NONE'])
+pseudo_op('POP_JUMP_IF_NOT_NONE', 265, ['POP_JUMP_FORWARD_IF_NOT_NONE', 'POP_JUMP_BACKWARD_IF_NOT_NONE'])
+pseudo_op('LOAD_METHOD', 266, ['LOAD_ATTR'])
+
+MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1
+
+del def_op, name_op, jrel_op, jabs_op, pseudo_op
+
+opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)]
+for op, i in opmap.items():
+    opname[i] = op
 
-del def_op, name_op, jrel_op, jabs_op
 
 _nb_ops = [
     ("NB_ADD", "+"),

Lib/dis.py

@@ -18,9 +18,10 @@ def test_stack_effect(self):
         self.assertRaises(ValueError, stack_effect, dis.opmap['BUILD_SLICE'])
         self.assertRaises(ValueError, stack_effect, dis.opmap['POP_TOP'], 0)
         # All defined opcodes
+        has_arg = dis.hasarg
         for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
             with self.subTest(opname=name):
-                if code < dis.HAVE_ARGUMENT:
+                if code not in has_arg:
                     stack_effect(code)
                     self.assertRaises(ValueError, stack_effect, code, 0)
                 else:
@@ -46,18 +47,20 @@ def test_stack_effect_jump(self):
         self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=True), 0)
         self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0)
         # All defined opcodes
+        has_arg = dis.hasarg
+        has_exc = dis.hasexc
         has_jump = dis.hasjabs + dis.hasjrel
         for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
             with self.subTest(opname=name):
-                if code < dis.HAVE_ARGUMENT:
+                if code not in has_arg:
                     common = stack_effect(code)
                     jump = stack_effect(code, jump=True)
                     nojump = stack_effect(code, jump=False)
                 else:
                     common = stack_effect(code, 0)
                     jump = stack_effect(code, 0, jump=True)
                     nojump = stack_effect(code, 0, jump=False)
-                if code in has_jump:
+                if code in has_jump or code in has_exc:
                     self.assertEqual(common, max(jump, nojump))
                 else:
                     self.assertEqual(jump, common)