From ca81806af3aaf7ec47e10017aa4bcaa182f3497b Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mishra Date: Sat, 11 Apr 2026 15:30:46 +0530 Subject: [PATCH 1/5] gh-148285: Allow recording uops after specializing uops --- Tools/cases_generator/analyzer.py | 34 +++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 6ba9c43ef1f0c3..af73090ad998a4 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1132,11 +1132,16 @@ def add_macro( macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] ) -> None: parts: list[Part] = [] - first = True + # Counts only real OpName entries (not CacheEffect/flush) so we + # know the exact position of each concrete uop inside the macro. + # CacheEffect → becomes Skip; flush → becomes Flush. + # Neither increments uop_index because neither is a "real" uop. + uop_index = 0 for part in macro.uops: match part: case parser.OpName(): if part.name == "flush": + # flush is structural, not a real uop; leave uop_index alone. parts.append(Flush()) else: if part.name not in uops: @@ -1144,13 +1149,30 @@ def add_macro( f"No Uop named {part.name}", macro.tokens[0] ) uop = uops[part.name] - if uop.properties.records_value and not first: - raise analysis_error( - f"Recording uop {part.name} must be first in macro", - macro.tokens[0]) + if uop.properties.records_value: + # A recording uop is legal in exactly two positions: + # 1. It is the very first real uop (uop_index == 0). + # 2. It is at index 1 AND the immediately preceding + # real uop is a specializing uop, identified by + # the "_SPECIALIZE_" name prefix. + # (Specializing uops are Tier-1-only; recording + # uops are Tier-2-only — they are orthogonal at + # runtime, so this ordering is safe.) + preceding_is_specializing = ( + uop_index == 1 + and isinstance(parts[-1], Uop) + and parts[-1].name.startswith("_SPECIALIZE_") + ) + if uop_index != 0 and not preceding_is_specializing: + raise analysis_error( + f"Recording uop {part.name} must be first in macro " + f"or immediately follow a specializing uop", + macro.tokens[0]) parts.append(uop) - first = False + uop_index += 1 case parser.CacheEffect(): + # Cache-entry skips are structural; they do not occupy a uop + # slot, so uop_index is not incremented. parts.append(Skip(part.size)) case _: assert False From 440b8230cb1e823d820811acba63ffb02562454a Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mishra Date: Sat, 11 Apr 2026 21:11:44 +0530 Subject: [PATCH 2/5] Apply reviewer suggestions, add tests, and regen cases --- Python/optimizer_bytecodes.c | 24388 +++++++++++++++++++++-- Tools/cases_generator/analyzer.py | 29 +- Tools/cases_generator/test_analyzer.py | 141 + 3 files changed, 22525 insertions(+), 2033 deletions(-) create mode 100644 Tools/cases_generator/test_analyzer.py diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index c12a4f4131bc7e..207c95f5dac597 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1,2290 +1,22654 @@ -#include "Python.h" -#include "pycore_optimizer.h" -#include "pycore_uops.h" -#include "pycore_uop_ids.h" -#include "internal/pycore_moduleobject.h" - -#define op(name, ...) /* NAME is ignored */ - -typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; - -/* Shortened forms for convenience */ -#define sym_is_not_null _Py_uop_sym_is_not_null -#define sym_is_const _Py_uop_sym_is_const -#define sym_get_const _Py_uop_sym_get_const -#define sym_new_unknown _Py_uop_sym_new_unknown -#define sym_new_not_null _Py_uop_sym_new_not_null -#define sym_new_type _Py_uop_sym_new_type -#define sym_is_null _Py_uop_sym_is_null -#define sym_new_const _Py_uop_sym_new_const -#define sym_new_null _Py_uop_sym_new_null -#define sym_matches_type _Py_uop_sym_matches_type -#define sym_matches_type_version _Py_uop_sym_matches_type_version -#define sym_get_type _Py_uop_sym_get_type -#define sym_has_type _Py_uop_sym_has_type -#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) -#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) -#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) -#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) -#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) -#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM) -#define sym_is_bottom _Py_uop_sym_is_bottom -#define frame_new _Py_uop_frame_new -#define frame_new_from_symbol _Py_uop_frame_new_from_symbol -#define frame_pop _Py_uop_frame_pop -#define sym_new_tuple _Py_uop_sym_new_tuple -#define sym_tuple_getitem _Py_uop_sym_tuple_getitem -#define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_symbol_is_immortal -#define sym_new_compact_int _Py_uop_sym_new_compact_int -#define sym_is_compact_int _Py_uop_sym_is_compact_int -#define sym_new_truthiness _Py_uop_sym_new_truthiness -#define sym_new_predicate _Py_uop_sym_new_predicate -#define sym_apply_predicate_narrowing _Py_uop_sym_apply_predicate_narrowing -#define sym_set_recorded_type(SYM, TYPE) _Py_uop_sym_set_recorded_type(ctx, SYM, TYPE) -#define sym_set_recorded_value(SYM, VAL) _Py_uop_sym_set_recorded_value(ctx, SYM, VAL) -#define sym_set_recorded_gen_func(SYM, VAL) _Py_uop_sym_set_recorded_gen_func(ctx, SYM, VAL) -#define sym_get_probable_func_code _Py_uop_sym_get_probable_func_code -#define sym_get_probable_value _Py_uop_sym_get_probable_value -#define sym_get_probable_type _Py_uop_sym_get_probable_type -#define sym_set_stack_depth(DEPTH, SP) _Py_uop_sym_set_stack_depth(ctx, DEPTH, SP) - -extern int -optimize_to_bool( - _PyUOpInstruction *this_instr, - JitOptContext *ctx, - JitOptSymbol *value, - JitOptSymbol **result_ptr, - uint16_t prefix, uint16_t suffix); - -extern void -eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit); - -extern PyCodeObject *get_code(_PyUOpInstruction *op); - -static int -dummy_func(void) { - - PyCodeObject *co; - int oparg; - JitOptSymbol *flag; - JitOptSymbol *left; - JitOptSymbol *right; - JitOptSymbol *value; - JitOptSymbol *res; - JitOptSymbol *iter; - JitOptSymbol *top; - JitOptSymbol *bottom; - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractFrame *new_frame; - JitOptContext *ctx; - _PyUOpInstruction *this_instr; - _PyBloomFilter *dependencies; - int modified; - int curr_space; - int max_space; - _PyUOpInstruction *first_valid_check_stack; - _PyUOpInstruction *corresponding_check_stack; - -// BEGIN BYTECODES // - - op(_MAKE_HEAP_SAFE, (value -- value)) { - // eliminate _MAKE_HEAP_SAFE when we *know* the value is immortal - if (sym_is_immortal(PyJitRef_Unwrap(value))) { - ADD_OP(_NOP, 0, 0); - } - value = PyJitRef_StripBorrowInfo(value); - } - - op(_COPY_FREE_VARS, (--)) { - PyCodeObject *co = get_current_code_object(ctx); - if (co == NULL) { - ctx->done = true; - break; - } - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - ctx->frame->locals[offset + i] = sym_new_not_null(ctx); - } - } - - op(_LOAD_FAST_CHECK, (-- value)) { - value = GETLOCAL(oparg); - // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { - ctx->done = true; - } - assert(!PyJitRef_IsUnique(value)); - } - - op(_LOAD_FAST, (-- value)) { - value = GETLOCAL(oparg); - assert(!PyJitRef_IsUnique(value)); - } - - op(_LOAD_FAST_BORROW, (-- value)) { - value = PyJitRef_Borrow(GETLOCAL(oparg)); - assert(!PyJitRef_IsUnique(value)); - } - - op(_LOAD_FAST_AND_CLEAR, (-- value)) { - value = GETLOCAL(oparg); - JitOptRef temp = sym_new_null(ctx); - GETLOCAL(oparg) = temp; - assert(!PyJitRef_IsUnique(value)); - } - - op(_GUARD_TYPE_VERSION_LOCKED, (type_version/2, owner -- owner)) { - assert(type_version); - if (sym_matches_type_version(owner, type_version)) { - ADD_OP(_NOP, 0, 0); - } else { - PyTypeObject *probable_type = sym_get_probable_type(owner); - if (probable_type->tp_version_tag == type_version && sym_set_type_version(owner, type_version)) { - // Promote the probable type version to a known one. - if ((probable_type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) { - PyType_Watch(TYPE_WATCHER_ID, (PyObject *)probable_type); - _Py_BloomFilter_Add(dependencies, probable_type); - } - } - } - } - - op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner -- o)) { - (void)offset; - (void)value; - o = owner; - } - - op(_STORE_ATTR_WITH_HINT, (hint/1, value, owner -- o)) { - (void)hint; - (void)value; - o = owner; - } - - op(_SWAP_FAST, (value -- trash)) { - JitOptRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyJitRef_RemoveUnique(value); - trash = tmp; - } - - op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- ls, ss)) { - (void)value; - ls = list_st; - ss = sub_st; - } - - op(_STORE_ATTR_SLOT, (index/1, value, owner -- o)) { - (void)index; - (void)value; - o = owner; - } - - op(_STORE_SUBSCR_DICT, (value, dict_st, sub -- st)) { - PyObject *sub_o = sym_get_const(ctx, sub); - if (sub_o != NULL) { - optimize_dict_known_hash(ctx, dependencies, this_instr, - sub_o, _STORE_SUBSCR_DICT_KNOWN_HASH); +// This file is generated by Tools/cases_generator/tier2_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 + + case _NOP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _NOP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _NOP_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _NOP_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_PERIODIC_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 because it is replaced */ + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _RESUME_CHECK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _RESUME_CHECK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _RESUME_CHECK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _RESUME_CHECK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _LOAD_FAST_CHECK_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + value = PyStackRef_DUP(value_s); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_CHECK_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + value = PyStackRef_DUP(value_s); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_CHECK_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + value = PyStackRef_DUP(value_s); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_4_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_4_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_5_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_5_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_6_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_6_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_7_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_7_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_4_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_4_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_5_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_5_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_6_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_6_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_7_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_7_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_AND_CLEAR_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_AND_CLEAR_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FAST_AND_CLEAR_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_CONST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_CONST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_CONST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SMALL_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_0_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_0_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_0_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_1_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_1_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_3_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_3_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_5_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_5_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_5_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_6_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_6_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_6_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_7_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_7_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_7_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = CURRENT_OPARG(); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_NOP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_NOP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_NOP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_NOP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_INT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_INT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_INT_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_FLOAT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_FLOAT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_FLOAT_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_UNICODE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_UNICODE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_UNICODE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_UNICODE_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_TOP_OPARG_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_NULL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef res; + res = PyStackRef_NULL; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_NULL_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + res = PyStackRef_NULL; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_NULL_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + res = PyStackRef_NULL; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _END_FOR_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_ITER_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef index_or_null; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + index_or_null = _stack_item_1; + iter = _stack_item_0; + (void)index_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _END_SEND_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef index_or_null; + _PyStackRef receiver; + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + index_or_null = _stack_item_1; + receiver = _stack_item_0; + val = value; + (void)index_or_null; + stack_pointer[0] = val; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = val; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NEGATIVE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NEGATIVE_FLOAT_INPLACE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + value = stack_pointer[-1]; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyFloat_CheckExact(val_o)); + assert(_PyObject_IsUniquelyReferenced(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + double dres = -((PyFloatObject *)val_o)->ob_fval; + ((PyFloatObject *)val_o)->ob_fval = dres; + res = value; + v = PyStackRef_NULL; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NEGATIVE_FLOAT_INPLACE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyFloat_CheckExact(val_o)); + assert(_PyObject_IsUniquelyReferenced(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + double dres = -((PyFloatObject *)val_o)->ob_fval; + ((PyFloatObject *)val_o)->ob_fval = dres; + res = value; + v = PyStackRef_NULL; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NEGATIVE_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyFloat_CheckExact(val_o)); + assert(_PyObject_IsUniquelyReferenced(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + double dres = -((PyFloatObject *)val_o)->ob_fval; + ((PyFloatObject *)val_o)->ob_fval = dres; + res = value; + v = PyStackRef_NULL; + _tos_cache2 = v; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NOT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NOT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NOT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_NOT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = err ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_BOOL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_BOOL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_BOOL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_BOOL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_INT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + value = stack_pointer[-1]; + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_INT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; + v = value; + _tos_cache2 = v; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_LIST_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_LIST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_LIST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SLICE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SLICE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SLICE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SLICE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_LIST_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_LIST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_LIST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + v = value; + _tos_cache2 = v; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_NONE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_NONE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_NONE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_NONE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_COMPACT_ASCII_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_IS_COMPACT_ASCII(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_COMPACT_ASCII_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_IS_COMPACT_ASCII(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_COMPACT_ASCII_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_IS_COMPACT_ASCII(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_COMPACT_ASCII_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_IS_COMPACT_ASCII(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_UNICODE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_UNICODE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_UNICODE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_UNICODE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_UNICODE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_UNICODE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_UNICODE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_UNICODE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_STR_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + value = stack_pointer[-1]; + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_STR_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _TO_BOOL_STR_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; + v = value; + _tos_cache2 = v; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _REPLACE_WITH_TRUE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + value = stack_pointer[-1]; + res = PyStackRef_True; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _REPLACE_WITH_TRUE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + res = PyStackRef_True; + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _REPLACE_WITH_TRUE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + res = PyStackRef_True; + v = value; + _tos_cache2 = v; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNARY_INVERT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + v = value; + _tos_cache1 = v; + _tos_cache0 = res; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_INT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_INT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_INT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_OVERFLOWED_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_OVERFLOWED_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_OVERFLOWED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_OVERFLOWED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_OVERFLOWED_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_OVERFLOWED_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_OVERFLOWED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_OVERFLOWED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); + if (PyStackRef_IsNull(_int_inplace_res)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + res = _int_inplace_res; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_FLOAT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_FLOAT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, left, +); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, left, +); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, left, +); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, left, -); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, left, -); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, left, -); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, left, *); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, left, *); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, left, *); + res = left; + l = PyStackRef_NULL; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, right, +); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, right, +); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, right, +); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, right, *); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, right, *); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, right, *); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + FLOAT_INPLACE_OP(left, right, right, -); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + FLOAT_INPLACE_OP(left, right, right, -); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + FLOAT_INPLACE_OP(left, right, right, -); + res = right; + l = left; + r = PyStackRef_NULL; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_UNICODE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_UNICODE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + stack_pointer[0] = right; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_ADD_UNICODE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_INPLACE_ADD_UNICODE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0_16(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyUnicode_Append(&temp, right_o); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + stack_pointer = _PyFrame_GetStackPointer(frame); + *target_local = PyStackRef_NULL; + if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(temp); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_EXTEND_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_EXTEND_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_SLICE_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stop = _stack_item_2; + start = _stack_item_1; + container = _stack_item_0; + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *res_o; + if (PyList_CheckExact(container_o)) { + stack_pointer[0] = container; + stack_pointer[1] = start; + stack_pointer[2] = stop; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyList_BinarySlice(container_o, start_o, stop_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else if (PyTuple_CheckExact(container_o)) { + stack_pointer[0] = container; + stack_pointer[1] = start; + stack_pointer[2] = stop; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else if (PyUnicode_CheckExact(container_o)) { + stack_pointer[0] = container; + stack_pointer[1] = start; + stack_pointer[2] = stop; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + PyObject *slice = PySlice_New(start_o, stop_o, NULL); + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer[0] = container; + stack_pointer[1] = start; + stack_pointer[2] = stop; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(container_o, slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + } + stack_pointer += 3; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = stop; + stop = PyStackRef_NULL; + stack_pointer[-3] = container; + stack_pointer[-2] = start; + stack_pointer[-1] = stop; + PyStackRef_CLOSE(tmp); + tmp = start; + start = PyStackRef_NULL; + stack_pointer[-2] = start; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_SLICE_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stop = _stack_item_2; + start = _stack_item_1; + container = _stack_item_0; + v = stack_pointer[-1]; + stack_pointer[0] = container; + stack_pointer[1] = start; + stack_pointer[2] = stop; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -4; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_LIST_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + _PyStackRef ls; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + list_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyList_GET_SIZE(list); + } + #ifdef Py_GIL_DISABLED + stack_pointer[0] = list_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = list_st; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index < 0 || index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = list_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += 2; + #endif + STAT_INC(BINARY_OP, hit); + ls = list_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ls; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_LIST_SLICE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + _PyStackRef ls; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + list_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + stack_pointer[0] = list_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_SliceSubscript(list, sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(BINARY_OP, hit); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + ls = list_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ls; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_STR_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; + _PyStackRef s; + _PyStackRef i; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + str_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; + assert(c < 128); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + s = str_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + _tos_cache2 = i; + _tos_cache1 = s; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_USTR_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; + _PyStackRef s; + _PyStackRef i; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + str_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = str_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + s = str_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + _tos_cache2 = i; + _tos_cache1 = s; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TUPLE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_TUPLE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef _stack_item_0 = _tos_cache0; + sub_st = _stack_item_0; + tuple_st = stack_pointer[-1]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = sub_st; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = sub_st; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + tuple_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub_st = _stack_item_2; + tuple_st = _stack_item_1; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = tuple_st; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = tuple_st; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = sub_st; + _tos_cache1 = tuple_st; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + _PyStackRef ts; + _PyStackRef ss; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + ts = tuple_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ts; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + _PyStackRef ts; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + sub_st = _stack_item_0; + tuple_st = stack_pointer[-1]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + ts = tuple_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ts; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + _PyStackRef ts; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + tuple_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + ts = tuple_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ts; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_ANY_DICT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_ANY_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_ANY_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_ANY_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENDICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENDICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENDICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENDICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + _PyStackRef ds; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + dict_st = _stack_item_0; + PyObject *hash = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyAnyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + stack_pointer[0] = dict_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = _PyDict_GetItemRef_KnownHash((PyDictObject *)dict, sub, (Py_hash_t)hash, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (rc <= 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ds; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + _PyStackRef ds; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + dict_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyAnyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + stack_pointer[0] = dict_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (rc <= 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ds; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_CHECK_FUNC_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + container = _stack_item_0; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + _tos_cache2 = getitem; + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_INIT_CALL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef new_frame; + getitem = stack_pointer[-1]; + sub = stack_pointer[-2]; + container = stack_pointer[-3]; + STAT_INC(BINARY_OP, hit); + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_INIT_CALL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + getitem = _stack_item_0; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + STAT_INC(BINARY_OP, hit); + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_INIT_CALL_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + getitem = _stack_item_1; + sub = _stack_item_0; + container = stack_pointer[-1]; + STAT_INC(BINARY_OP, hit); + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_INIT_CALL_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + getitem = _stack_item_2; + sub = _stack_item_1; + container = _stack_item_0; + STAT_INC(BINARY_OP, hit); + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LIST_APPEND_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef v; + _PyStackRef list; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + list = stack_pointer[-1 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SET_ADD_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef v; + _PyStackRef set; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + set = stack_pointer[-1 - (oparg-1)]; + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_SUBSCR_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub = _stack_item_2; + container = _stack_item_1; + v = _stack_item_0; + stack_pointer[0] = v; + stack_pointer[1] = container; + stack_pointer[2] = sub; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_SUBSCR_LIST_INT_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; + _PyStackRef ls; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub_st = _stack_item_2; + list_st = _stack_item_1; + value = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (!LOCK_OBJECT(list)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = list_st; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t len = PyList_GET_SIZE(list); + if (index < 0) { + index += len; + } + if (index < 0 || index >= len) { + UNLOCK_OBJECT(list); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = list_st; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + ls = list_st; + ss = sub_st; + stack_pointer[0] = ls; + stack_pointer[1] = ss; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache1 = ss; + _tos_cache0 = ls; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_SUBSCR_DICT_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub; + _PyStackRef dict_st; + _PyStackRef value; + _PyStackRef st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub = _stack_item_2; + dict_st = _stack_item_1; + value = _stack_item_0; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + stack_pointer[0] = value; + stack_pointer[1] = dict_st; + stack_pointer[2] = sub; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + st = dict_st; + _tos_cache0 = st; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_SUBSCR_DICT_KNOWN_HASH_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub; + _PyStackRef dict_st; + _PyStackRef value; + _PyStackRef st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub = _stack_item_2; + dict_st = _stack_item_1; + value = _stack_item_0; + PyObject *hash = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + stack_pointer[0] = value; + stack_pointer[1] = dict_st; + stack_pointer[2] = sub; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2_KnownHash((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value), + (Py_hash_t)hash); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + st = dict_st; + _tos_cache0 = st; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DELETE_SUBSCR_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub; + _PyStackRef container; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub = _stack_item_1; + container = _stack_item_0; + stack_pointer[0] = container; + stack_pointer[1] = sub; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_INTRINSIC_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = _stack_item_0; + assert(oparg <= MAX_INTRINSIC_1); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + v = value; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache1 = v; + _tos_cache0 = res; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_INTRINSIC_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value1_st; + _PyStackRef value2_st; + _PyStackRef res; + _PyStackRef vs1; + _PyStackRef vs2; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value1_st = _stack_item_1; + value2_st = _stack_item_0; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + stack_pointer[0] = value2_st; + stack_pointer[1] = value1_st; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + vs1 = value1_st; + vs2 = value2_st; + _tos_cache2 = vs2; + _tos_cache1 = vs1; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAKE_HEAP_SAFE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + value = PyStackRef_MakeHeapSafe(value); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAKE_HEAP_SAFE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + value = PyStackRef_MakeHeapSafe(value); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAKE_HEAP_SAFE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + value = PyStackRef_MakeHeapSafe(value); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAKE_HEAP_SAFE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + value = PyStackRef_MakeHeapSafe(value); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _RETURN_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef retval; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + retval = _stack_item_0; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GET_AITER_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef obj; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + obj = _stack_item_0; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + stack_pointer[0] = obj; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + stack_pointer[0] = obj; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _tos_cache0 = iter; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GET_ANEXT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef aiter; + _PyStackRef awaitable; + _PyStackRef _stack_item_0 = _tos_cache0; + aiter = _stack_item_0; + stack_pointer[0] = aiter; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + _tos_cache1 = awaitable; + _tos_cache0 = aiter; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GET_AWAITABLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + iterable = _stack_item_0; + stack_pointer[0] = iterable; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _tos_cache0 = iter; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _SEND_GEN_FRAME_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + v = _stack_item_2; + receiver = _stack_item_0; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = v; + _tos_cache1 = _stack_item_1; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = v; + _tos_cache1 = _stack_item_1; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_1; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _YIELD_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef retval; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + retval = _stack_item_0; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + _PyStackRef temp = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _POP_EXCEPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef exc_value; + _PyStackRef _stack_item_0 = _tos_cache0; + exc_value = _stack_item_0; + _PyErr_StackItem *exc_info = tstate->exc_info; + stack_pointer[0] = exc_value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_COMMON_CONSTANT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_COMMON_CONSTANT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_COMMON_CONSTANT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_BUILD_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bc; + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + _tos_cache0 = bc; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_NAME_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (PyDict_CheckExact(ns)) { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DELETE_NAME_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef *top; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + top = &stack_pointer[oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_TWO_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[0] = val1; + stack_pointer[1] = val0; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache1 = val0; + _tos_cache0 = val1; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + seq = stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache1 = val0; + _tos_cache0 = val1; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + seq = _stack_item_0; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache1 = val0; + _tos_cache0 = val1; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + seq = _stack_item_1; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache2 = val0; + _tos_cache1 = val1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val2; + _PyStackRef val1; + _PyStackRef val0; + seq = stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + val2 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 2)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache2 = val0; + _tos_cache1 = val1; + _tos_cache0 = val2; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef val2; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + seq = _stack_item_0; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + val2 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 2)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + _tos_cache2 = val0; + _tos_cache1 = val1; + _tos_cache0 = val2; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_TUPLE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + values = &stack_pointer[0]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + values = &stack_pointer[0]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + assert(PyTuple_CheckExact(seq_o)); + assert(PyTuple_GET_SIZE(seq_o) == oparg); + assert(_PyObject_IsUniquelyReferenced(seq_o)); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectSteal(items[i]); + } + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_LIST_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + values = &stack_pointer[0]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _UNPACK_EX_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef seq; + _PyStackRef *top; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + top = &stack_pointer[1 + (oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_ATTR_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + v = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = v; + stack_pointer[1] = owner; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DELETE_ATTR_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = owner; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_GLOBAL_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DELETE_GLOBAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_LOCALS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + locals = PyStackRef_FromPyObjectNew(l); + _tos_cache0 = locals; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_LOCALS_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef locals; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *l = LOCALS(); + if (l == NULL) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + locals = PyStackRef_FromPyObjectNew(l); + _tos_cache1 = locals; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_LOCALS_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef locals; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *l = LOCALS(); + if (l == NULL) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + locals = PyStackRef_FromPyObjectNew(l); + _tos_cache2 = locals; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + + case _LOAD_NAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef v; + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (PyLazyImport_CheckExact(v_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (l_v == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(v_o); + Py_DECREF(l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(v_o, l_v); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + v = PyStackRef_FromPyObjectSteal(v_o); + _tos_cache0 = v; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_GLOBAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *res; + oparg = CURRENT_OPARG(); + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_NULL_CONDITIONAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *null; + oparg = CURRENT_OPARG(); + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_GLOBAL_MODULE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef res; + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_GLOBAL_BUILTINS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef res; + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DELETE_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAKE_CELL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DELETE_DEREF_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_FROM_DICT_OR_DEREF_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef class_dict_st; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + class_dict_st = _stack_item_0; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + int err; + stack_pointer[0] = class_dict_st; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_DEREF_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = _PyCell_GetStackRef(cell); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_DEREF_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COPY_FREE_VARS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COPY_FREE_VARS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COPY_FREE_VARS_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COPY_FREE_VARS_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_STRING_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *pieces; + _PyStackRef str; + oparg = CURRENT_OPARG(); + pieces = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + str = PyStackRef_FromPyObjectSteal(str_o); + _tos_cache0 = str; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_INTERPOLATION_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *format; + _PyStackRef str; + _PyStackRef value; + _PyStackRef interpolation; + oparg = CURRENT_OPARG(); + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(format[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (interpolation_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + _tos_cache0 = interpolation; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_TEMPLATE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef interpolations; + _PyStackRef strings; + _PyStackRef template; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + interpolations = _stack_item_1; + strings = _stack_item_0; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + stack_pointer[0] = strings; + stack_pointer[1] = interpolations; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(interpolations); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(strings); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (template_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + template = PyStackRef_FromPyObjectSteal(template_o); + _tos_cache0 = template; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_TUPLE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *values; + _PyStackRef tup; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + _tos_cache0 = tup; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_LIST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *values; + _PyStackRef list; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (list_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + _tos_cache0 = list; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LIST_EXTEND_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iterable_st; + _PyStackRef list_st; + _PyStackRef i; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + iterable_st = _stack_item_0; + list_st = stack_pointer[-1 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + stack_pointer[0] = iterable_st; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(Py_IsNone(none_val)); + i = iterable_st; + _tos_cache0 = i; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SET_UPDATE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iterable; + _PyStackRef set; + _PyStackRef i; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + iterable = _stack_item_0; + set = stack_pointer[-1 - (oparg-1)]; + stack_pointer[0] = iterable; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + i = iterable; + _tos_cache0 = i; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_SET_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *values; + _PyStackRef set; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + if (err) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + _tos_cache0 = set; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_MAP_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *values; + _PyStackRef map; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg*2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + _tos_cache0 = map; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SETUP_ANNOTATIONS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DICT_UPDATE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef update; + _PyStackRef dict; + _PyStackRef upd; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + update = _stack_item_0; + dict = stack_pointer[-1 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + stack_pointer[0] = update; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (has_keys == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a mapping", + update_o); + Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_ChainExceptions1(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + upd = update; + _tos_cache0 = upd; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _DICT_MERGE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef update; + _PyStackRef dict; + _PyStackRef callable; + _PyStackRef u; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + update = _stack_item_0; + dict = stack_pointer[-1 - (oparg - 1)]; + callable = stack_pointer[-4 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + PyObject *dupkey = NULL; + stack_pointer[0] = update; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + Py_XDECREF(dupkey); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + u = update; + _tos_cache0 = u; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAP_ADD_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef key; + _PyStackRef dict_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = _stack_item_1; + key = _stack_item_0; + dict_st = stack_pointer[-1 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + stack_pointer[0] = key; + stack_pointer[1] = value; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SUPER_ATTR_ATTR_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + self_st = _stack_item_2; + class_st = _stack_item_1; + global_super_st = _stack_item_0; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = self_st; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = self_st; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer[0] = global_super_st; + stack_pointer[1] = class_st; + stack_pointer[2] = self_st; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (attr == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + _tos_cache0 = attr_st; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TYPE_VERSION_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); + assert(type_version != 0); + if (!PyType_Check((PyObject *)tp)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TYPE_VERSION_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); + assert(type_version != 0); + if (!PyType_Check((PyObject *)tp)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TYPE_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); + assert(type_version != 0); + if (!PyType_Check((PyObject *)tp)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_TYPE_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); + assert(type_version != 0); + if (!PyType_Check((PyObject *)tp)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_LOAD_SUPER_ATTR_METHOD_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef class_st; + _PyStackRef global_super_st; + oparg = CURRENT_OPARG(); + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_LOAD_SUPER_ATTR_METHOD_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + class_st = stack_pointer[-1]; + global_super_st = stack_pointer[-2]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_LOAD_SUPER_ATTR_METHOD_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + class_st = _stack_item_0; + global_super_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = class_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = class_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_LOAD_SUPER_ATTR_METHOD_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + class_st = _stack_item_1; + global_super_st = _stack_item_0; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = class_st; + _tos_cache0 = global_super_st; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SUPER_ATTR_METHOD_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr; + _PyStackRef self_or_null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + self_st = _stack_item_2; + class_st = _stack_item_1; + global_super_st = _stack_item_0; + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + PyObject *attr_o; + { + int *method_found_ptr = &method_found; + stack_pointer[0] = global_super_st; + stack_pointer[1] = class_st; + stack_pointer[2] = self_st; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (attr_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + attr = PyStackRef_FromPyObjectSteal(attr_o); + _tos_cache1 = self_or_null; + _tos_cache0 = attr; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *self_or_null; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + self_or_null = &stack_pointer[1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + stack_pointer[0] = owner; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(attr)) { + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } + else { + stack_pointer[0] = owner; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(attr)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + stack_pointer += -(oparg&1); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_LOCKED_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_LOCKED_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_LOCKED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_LOCKED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + owner = stack_pointer[-1]; + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache1 = o; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache1 = o; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache2 = o; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_MODULE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t dict_version = (uint32_t)CURRENT_OPERAND0_32(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache1 = o; + _tos_cache0 = attr; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_WITH_HINT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + o = owner; + _tos_cache1 = o; + _tos_cache0 = attr; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_SLOT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache1 = o; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_SLOT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache1 = o; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_SLOT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache2 = o; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_CLASS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[0] = owner; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_PROPERTY_FRAME_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *fget = (PyObject *)CURRENT_OPERAND0_64(); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_kwonlyargcount) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_argcount != 1) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */ + + case _GUARD_DORV_NO_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_DORV_NO_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_DORV_NO_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_DORV_NO_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_ATTR_INSTANCE_VALUE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + value = _stack_item_0; + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + o = owner; + stack_pointer[0] = o; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = o; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOCK_OBJECT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOCK_OBJECT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOCK_OBJECT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOCK_OBJECT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_ATTR_WITH_HINT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + value = _stack_item_0; + uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!LOCK_OBJECT(dict)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + stack_pointer[0] = value; + stack_pointer[1] = owner; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + o = owner; + stack_pointer[-2] = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = o; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _STORE_ATTR_SLOT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + value = _stack_item_0; + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + o = owner; + stack_pointer[0] = o; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = o; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COMPARE_OP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COMPARE_OP_FLOAT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + oparg = CURRENT_OPARG(); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + l = left; + r = right; + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COMPARE_OP_FLOAT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + l = left; + r = right; + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COMPARE_OP_FLOAT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + l = left; + r = right; + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COMPARE_OP_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + l = left; + r = right; + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _COMPARE_OP_STR_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + l = left; + r = right; + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _IS_OP_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + oparg = CURRENT_OPARG(); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _IS_OP_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + right = _stack_item_0; + left = stack_pointer[-1]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _IS_OP_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CONTAINS_OP_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SET_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_SET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENSET_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenSet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENSET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenSet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENSET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenSet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_TOS_FROZENSET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyFrozenSet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CONTAINS_OP_SET_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CONTAINS_OP_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_EG_MATCH_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef match_type_st; + _PyStackRef exc_value_st; + _PyStackRef rest; + _PyStackRef match; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + match_type_st = _stack_item_1; + exc_value_st = _stack_item_0; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + stack_pointer[0] = exc_value_st; + stack_pointer[1] = match_type_st; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + _tos_cache1 = match; + _tos_cache0 = rest; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_EXC_MATCH_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = b; + _tos_cache0 = left; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _IMPORT_NAME_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef fromlist; + _PyStackRef level; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + fromlist = _stack_item_1; + level = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *res_o; + if (!(oparg & 0x02)) { + stack_pointer[0] = level; + stack_pointer[1] = fromlist; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level), + oparg & 0x01); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer[0] = level; + stack_pointer[1] = fromlist; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _IMPORT_FROM_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef from; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + from = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o; + if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { + stack_pointer[0] = from; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_LazyImportFrom( + tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer[0] = from; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyEval_ImportFrom( + tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (res_o == NULL) { + stack_pointer[-1] = from; + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache1 = res; + _tos_cache0 = from; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ + + case _IS_NONE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + } + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ + + case _GET_LEN_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef obj; + _PyStackRef len; + _PyStackRef _stack_item_0 = _tos_cache0; + obj = _stack_item_0; + stack_pointer[0] = obj; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + len = PyStackRef_FromPyObjectSteal(len_o); + _tos_cache1 = len; + _tos_cache0 = obj; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_CLASS_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef names; + _PyStackRef type; + _PyStackRef subject; + _PyStackRef attrs; + _PyStackRef s; + _PyStackRef tp; + _PyStackRef n; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + names = _stack_item_2; + type = _stack_item_1; + subject = _stack_item_0; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + stack_pointer[0] = subject; + stack_pointer[1] = type; + stack_pointer[2] = names; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + attrs = PyStackRef_None; + } + s = subject; + tp = type; + n = names; + _tos_cache2 = n; + _tos_cache1 = tp; + _tos_cache0 = s; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[-3] = attrs; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_MAPPING_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_MAPPING_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + subject = _stack_item_0; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_MAPPING_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + subject = _stack_item_1; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = subject; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_SEQUENCE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_SEQUENCE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + subject = _stack_item_0; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_SEQUENCE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + subject = _stack_item_1; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = subject; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MATCH_KEYS_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef keys; + _PyStackRef subject; + _PyStackRef values_or_none; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + keys = _stack_item_1; + subject = _stack_item_0; + stack_pointer[0] = subject; + stack_pointer[1] = keys; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + _tos_cache2 = values_or_none; + _tos_cache1 = keys; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GET_ITER_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + iterable = _stack_item_0; + #ifdef Py_STATS + _Py_GatherStats_GetIter(iterable); + #endif + stack_pointer[0] = iterable; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsError(result)) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = result; + _tos_cache1 = index_or_null; + _tos_cache0 = iter; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ + + case _FOR_ITER_TIER_TWO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + stack_pointer[0] = iter; + stack_pointer[1] = null_or_index; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + stack_pointer[-1] = null_or_index; + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + } + next = item; + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ + + case _ITER_CHECK_LIST_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_LIST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_LIST_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ + + case _ITER_NEXT_LIST_TIER_TWO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + stack_pointer[0] = iter; + stack_pointer[1] = null_or_index; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result <= 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + #else + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + stack_pointer += 2; + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_TUPLE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_TUPLE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_NEXT_TUPLE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_NEXT_TUPLE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_NEXT_TUPLE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_RANGE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_RANGE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_RANGE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_CHECK_RANGE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_RANGE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_RANGE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_RANGE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_RANGE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_NEXT_RANGE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef next; + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + next = PyStackRef_FromPyObjectSteal(res); + _tos_cache2 = next; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_NEXT_RANGE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + next = PyStackRef_FromPyObjectSteal(res); + _tos_cache2 = next; + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _ITER_NEXT_RANGE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + stack_pointer[0] = iter; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + next = PyStackRef_FromPyObjectSteal(res); + _tos_cache2 = next; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _FOR_ITER_GEN_FRAME_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef gen_frame; + oparg = CURRENT_OPARG(); + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _FOR_ITER_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _FOR_ITER_GEN_FRAME_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + iter = _stack_item_0; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INSERT_NULL_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self; + _PyStackRef *method_and_self; + _PyStackRef _stack_item_0 = _tos_cache0; + self = _stack_item_0; + method_and_self = &stack_pointer[0]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_SPECIAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *method_and_self; + oparg = CURRENT_OPARG(); + method_and_self = &stack_pointer[-2]; + PyObject *name = _Py_SpecialMethods[oparg].name; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) + ? _Py_SpecialMethods[oparg].error_suggestion + : _Py_SpecialMethods[oparg].error; + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + assert(errfmt != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _WITH_EXCEPT_START_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef val; + _PyStackRef lasti; + _PyStackRef exit_self; + _PyStackRef exit_func; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + val = _stack_item_2; + lasti = _stack_item_0; + exit_self = stack_pointer[-1]; + exit_func = stack_pointer[-2]; + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + PyObject *original_tb = tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + assert(PyStackRef_IsTaggedInt(lasti)); + (void)lasti; + PyObject* res_o; + { + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + stack_pointer[0] = lasti; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = val; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(original_tb); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = res; + _tos_cache1 = val; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_EXC_INFO_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + _tos_cache1 = new_exc; + _tos_cache0 = prev_exc; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_EXC_INFO_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + _PyStackRef _stack_item_0 = _tos_cache0; + exc = _stack_item_0; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + _tos_cache1 = new_exc; + _tos_cache0 = prev_exc; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_EXC_INFO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + exc = _stack_item_1; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + _tos_cache2 = new_exc; + _tos_cache1 = prev_exc; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_KEYS_VERSION_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_KEYS_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_KEYS_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_KEYS_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAYBE_EXPAND_METHOD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _PY_FRAME_GENERAL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + new_frame = PyStackRef_Wrap(temp); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_METHOD_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _EXPAND_METHOD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_NON_PY_GENERAL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_PEP_523_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + if (IS_PEP523_HOOKED(tstate)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_PEP_523_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (IS_PEP523_HOOKED(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_PEP_523_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (IS_PEP523_HOOKED(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_PEP_523_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (IS_PEP523_HOOKED(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_STACK_SPACE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_REMAINING_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_REMAINING_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_REMAINING_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_REMAINING_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PUSH_FRAME_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + new_frame = _stack_item_0; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NULL_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NULL_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + null = stack_pointer[-1]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NULL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_0; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_1; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NOT_NULL_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NOT_NULL_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NOT_NULL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_NOS_NOT_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_THIRD_NULL_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_THIRD_NULL_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_THIRD_NULL_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = stack_pointer[-1]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_THIRD_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_0; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TYPE_1_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TYPE_1_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + callable = stack_pointer[-2]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TYPE_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + callable = stack_pointer[-1]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TYPE_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_TYPE_1_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + a = arg; + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_TYPE_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + arg = _stack_item_0; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + a = arg; + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_TYPE_1_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + arg = _stack_item_1; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + a = arg; + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_TYPE_1_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + arg = _stack_item_2; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + a = arg; + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_STR_1_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_STR_1_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + callable = stack_pointer[-2]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_STR_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + callable = stack_pointer[-1]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_STR_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_STR_1_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + arg = _stack_item_2; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + a = arg; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache1 = a; + _tos_cache0 = res; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TUPLE_1_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TUPLE_1_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + callable = stack_pointer[-2]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TUPLE_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + callable = stack_pointer[-1]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_TUPLE_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_TUPLE_1_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + arg = _stack_item_2; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + a = arg; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache1 = a; + _tos_cache0 = res; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_AND_ALLOCATE_OBJECT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + self_or_null = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CREATE_INIT_FRAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self; + _PyStackRef init; + _PyStackRef init_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + init = stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + shim->localsplus[0] = PyStackRef_DUP(self); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init, NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); + _tos_cache0 = init_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _EXIT_INIT_CHECK_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef should_be_none; + _PyStackRef _stack_item_0 = _tos_cache0; + should_be_none = _stack_item_0; + if (!PyStackRef_IsNone(should_be_none)) { + stack_pointer[0] = should_be_none; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_BUILTIN_CLASS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_vectorcall == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_BUILTIN_CLASS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_CallBuiltinClass_StackRef( + callable, + arguments, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_BUILTIN_O_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_BUILTIN_O_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = args[0]; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = s; + _tos_cache1 = c; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_BUILTIN_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_BUILTIN_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_BuiltinCallFast_StackRef( + callable, + arguments, + total_args + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LEN_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LEN_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + callable = stack_pointer[-2]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LEN_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + callable = stack_pointer[-1]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LEN_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_LEN_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + arg = _stack_item_2; + callable = _stack_item_0; + STAT_INC(CALL, hit); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + stack_pointer[0] = callable; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + a = arg; + c = callable; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = c; + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_ISINSTANCE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = stack_pointer[-3]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_ISINSTANCE_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_ISINSTANCE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + callable = stack_pointer[-2]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_ISINSTANCE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = stack_pointer[-1]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_ISINSTANCE_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef cls; + _PyStackRef instance; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + cls = _stack_item_2; + instance = _stack_item_1; + null = _stack_item_0; + callable = stack_pointer[-1]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + stack_pointer[0] = null; + stack_pointer[1] = instance; + stack_pointer[2] = cls; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(inst_o, cls_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + (void)null; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LIST_APPEND_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LIST_APPEND_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + callable = stack_pointer[-2]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LIST_APPEND_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + callable = stack_pointer[-1]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_LIST_APPEND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_LIST_APPEND_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + _PyStackRef none; + _PyStackRef c; + _PyStackRef s; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!LOCK_OBJECT(self_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = self; + none = PyStackRef_None; + _tos_cache2 = s; + _tos_cache1 = c; + _tos_cache0 = none; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_LIST_APPEND_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + _PyStackRef none; + _PyStackRef c; + _PyStackRef s; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + arg = _stack_item_0; + self = stack_pointer[-1]; + callable = stack_pointer[-2]; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!LOCK_OBJECT(self_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = arg; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + if (err) { + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = self; + none = PyStackRef_None; + _tos_cache2 = s; + _tos_cache1 = c; + _tos_cache0 = none; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_LIST_APPEND_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + _PyStackRef none; + _PyStackRef c; + _PyStackRef s; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + arg = _stack_item_1; + self = _stack_item_0; + callable = stack_pointer[-1]; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!LOCK_OBJECT(self_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = arg; + _tos_cache0 = self; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + if (err) { + stack_pointer[0] = self; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = self; + none = PyStackRef_None; + _tos_cache2 = s; + _tos_cache1 = c; + _tos_cache0 = none; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_LIST_APPEND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + _PyStackRef none; + _PyStackRef c; + _PyStackRef s; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + arg = _stack_item_2; + self = _stack_item_1; + callable = _stack_item_0; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!LOCK_OBJECT(self_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = arg; + _tos_cache1 = self; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + if (err) { + stack_pointer[0] = callable; + stack_pointer[1] = self; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = self; + none = PyStackRef_None; + _tos_cache2 = s; + _tos_cache1 = c; + _tos_cache0 = none; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (method->d_method->ml_flags != METH_O) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_O_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef a; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + } + STAT_INC(CALL, hit); + PyCFunction cfunc = method->d_method->ml_meth; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = arguments[0]; + a = arguments[1]; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = a; + _tos_cache1 = s; + _tos_cache0 = c; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_RECURSION_LIMIT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (_Py_ReachedRecursionLimit(tstate)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_O_INLINE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef a; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + callable = stack_pointer[-1 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg == 2); + STAT_INC(CALL, hit); + volatile PyCFunction cfunc_v = (PyCFunction)cfunc; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(args[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = args[0]; + a = args[1]; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = a; + _tos_cache1 = s; + _tos_cache0 = c; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[-1 - oparg] = res; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( + callable, + cfunc, + self, + arguments, + total_args + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_st; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_st = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( + callable, + cfunc_v, + self, + args - 1, + oparg + 1 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (method->d_method->ml_flags != METH_NOARGS) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + STAT_INC(CALL, hit); + PyCFunction cfunc = method->d_method->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = args[0]; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = s; + _tos_cache1 = c; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + callable = stack_pointer[-1 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + assert(oparg == 1); + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + STAT_INC(CALL, hit); + volatile PyCFunction cfunc_v = (PyCFunction)cfunc; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + c = callable; + s = args[0]; + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = s; + _tos_cache1 = c; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (method->d_method->ml_flags != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( + callable, + cfunc, + self, + arguments, + total_args + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef self_st; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_st = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(self != NULL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( + callable, + cfunc_v, + self, + args - 1, + oparg + 1 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _MAYBE_EXPAND_METHOD_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + } + _tos_cache0 = _stack_item_0; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _PY_FRAME_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + kwnames = _stack_item_0; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[0] = kwnames; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + new_frame = PyStackRef_Wrap(temp); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_METHOD_VERSION_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _EXPAND_METHOD_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = _stack_item_0; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_KW_NON_PY_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + kwnames = _stack_item_0; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + stack_pointer[0] = kwnames; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _MAKE_CALLARGS_A_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef callargs; + _PyStackRef func; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callargs = _stack_item_1; + func = stack_pointer[-1]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = callargs; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = callargs; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _CHECK_IS_PY_CALLABLE_EX_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + func_st = stack_pointer[-4]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) != &PyFunction_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = stack_pointer[-3]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_PY_CALLABLE_EX_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + _PyStackRef _stack_item_0 = _tos_cache0; + func_st = stack_pointer[-3]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) != &PyFunction_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_PY_CALLABLE_EX_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + func_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) != &PyFunction_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_PY_CALLABLE_EX_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + func_st = stack_pointer[-1]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) != &PyFunction_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _PY_FRAME_EX_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef kwargs_st; + _PyStackRef callargs_st; + _PyStackRef func_st; + _PyStackRef ex_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + kwargs_st = _stack_item_2; + callargs_st = _stack_item_1; + func_st = stack_pointer[-1]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + assert(Py_TYPE(func) == &PyFunction_Type); + assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + ex_frame = PyStackRef_Wrap(new_frame); + _tos_cache0 = ex_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_EX_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + func_st = stack_pointer[-4]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = stack_pointer[-3]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_EX_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + _PyStackRef _stack_item_0 = _tos_cache0; + func_st = stack_pointer[-3]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_EX_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + func_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_EX_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + func_st = stack_pointer[-1]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CALL_FUNCTION_EX_NON_PY_GENERAL_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef kwargs_st; + _PyStackRef callargs_st; + _PyStackRef null; + _PyStackRef func_st; + _PyStackRef result; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + kwargs_st = _stack_item_2; + callargs_st = _stack_item_1; + null = _stack_item_0; + func_st = stack_pointer[-1]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + (void)null; + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[0] = null; + stack_pointer[1] = callargs_st; + stack_pointer[2] = kwargs_st; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + result = PyStackRef_FromPyObjectSteal(result_o); + _tos_cache0 = result; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - (void)value; - st = dict_st; - } - - op(_PUSH_NULL, (-- res)) { - res = PyJitRef_Borrow(sym_new_null(ctx)); - } - - op(_GUARD_TOS_INT, (value -- value)) { - if (sym_is_compact_int(value)) { - ADD_OP(_NOP, 0, 0); - } - else { - if (sym_get_type(value) == &PyLong_Type) { - ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); - } - sym_set_compact_int(value); - } - } - - op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - if (sym_is_compact_int(left)) { - ADD_OP(_NOP, 0, 0); - } - else { - if (sym_get_type(left) == &PyLong_Type) { - ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); - } - sym_set_compact_int(left); - } - } - - op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { - PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); - if (type) { - if (type == sym_get_const(ctx, owner)) { - ADD_OP(_NOP, 0, 0); + + case _MAKE_FUNCTION_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef codeobj_st; + _PyStackRef func; + _PyStackRef co; + _PyStackRef _stack_item_0 = _tos_cache0; + codeobj_st = _stack_item_0; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + stack_pointer[0] = codeobj_st; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } - else { - sym_set_const(owner, type); - } - } - } - - op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { - assert(type_version); - assert(this_instr[-1].opcode == _RECORD_TOS_TYPE); - if (sym_matches_type_version(owner, type_version)) { - ADD_OP(_NOP, 0, 0); - } else { - // add watcher so that whenever the type changes we invalidate this - PyTypeObject *type = _PyType_LookupByVersion(type_version); - // if the type is null, it was not found in the cache (there was a conflict) - // with the key, in which case we can't trust the version - if (type) { - // if the type version was set properly, then add a watcher - // if it wasn't this means that the type version was previously set to something else - // and we set the owner to bottom, so we don't need to add a watcher because we must have - // already added one earlier. - if (sym_set_type_version(owner, type_version)) { - PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); - _Py_BloomFilter_Add(dependencies, type); - } - } - } - } - - op(_GUARD_TOS_FLOAT, (value -- value)) { - if (sym_matches_type(value, &PyFloat_Type)) { - ADD_OP(_NOP, 0, 0); - } - sym_set_type(value, &PyFloat_Type); - } - - op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { - if (sym_matches_type(left, &PyFloat_Type)) { - ADD_OP(_NOP, 0, 0); - } - sym_set_type(left, &PyFloat_Type); - } - - op(_BINARY_OP, (lhs, rhs -- res, l, r)) { - l = lhs; - r = rhs; - REPLACE_OPCODE_IF_EVALUATES_PURE(lhs, rhs, res); - bool lhs_int = sym_matches_type(lhs, &PyLong_Type); - bool rhs_int = sym_matches_type(rhs, &PyLong_Type); - bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); - bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); - if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { - // There's something other than an int or float involved: - res = sym_new_unknown(ctx); - } - else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { - // This one's fun... the *type* of the result depends on the - // *values* being exponentiated. However, exponents with one - // constant part are reasonably common, so it's probably worth - // trying to infer some simple cases: - // - A: 1 ** 1 -> 1 (int ** int -> int) - // - B: 1 ** -1 -> 1.0 (int ** int -> float) - // - C: 1.0 ** 1 -> 1.0 (float ** int -> float) - // - D: 1 ** 1.0 -> 1.0 (int ** float -> float) - // - E: -1 ** 0.5 ~> 1j (int ** float -> complex) - // - F: 1.0 ** 1.0 -> 1.0 (float ** float -> float) - // - G: -1.0 ** 0.5 ~> 1j (float ** float -> complex) - if (rhs_float) { - // Case D, E, F, or G... can't know without the sign of the LHS - // or whether the RHS is whole, which isn't worth the effort: - res = sym_new_unknown(ctx); - } - else if (lhs_float) { - // Case C: - res = sym_new_type(ctx, &PyFloat_Type); - } - else if (!sym_is_const(ctx, rhs)) { - // Case A or B... can't know without the sign of the RHS: - res = sym_new_unknown(ctx); - } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { - // Case B: - res = sym_new_type(ctx, &PyFloat_Type); + co = codeobj_st; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + _tos_cache1 = co; + _tos_cache0 = func; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SET_FUNCTION_ATTRIBUTE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_in; + _PyStackRef attr_st; + _PyStackRef func_out; + oparg = CURRENT_OPARG(); + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + _tos_cache0 = func_out; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SET_FUNCTION_ATTRIBUTE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_in; + _PyStackRef attr_st; + _PyStackRef func_out; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + func_in = _stack_item_0; + attr_st = stack_pointer[-1]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + _tos_cache0 = func_out; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SET_FUNCTION_ATTRIBUTE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_in; + _PyStackRef attr_st; + _PyStackRef func_out; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + func_in = _stack_item_1; + attr_st = _stack_item_0; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + _tos_cache0 = func_out; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SET_FUNCTION_ATTRIBUTE_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef func_in; + _PyStackRef attr_st; + _PyStackRef func_out; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + func_in = _stack_item_2; + attr_st = _stack_item_1; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + _tos_cache1 = func_out; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _RETURN_GENERATOR_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BUILD_SLICE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef *args; + _PyStackRef slice; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (slice_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + slice = PyStackRef_FromPyObjectStealMortal(slice_o); + _tos_cache0 = slice; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CONVERT_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef result; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = _stack_item_0; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + result = PyStackRef_FromPyObjectSteal(result_o); + _tos_cache0 = result; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _FORMAT_SIMPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); } else { - // Case A: - res = sym_new_type(ctx, &PyLong_Type); + res = value; } + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = sym_new_type(ctx, &PyFloat_Type); + + case _FORMAT_WITH_SPEC_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef fmt_spec; + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + fmt_spec = _stack_item_1; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer[1] = fmt_spec; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (lhs_int && rhs_int) { - res = sym_new_type(ctx, &PyLong_Type); + + case _COPY_1_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1]; + top = PyStackRef_DUP(bottom); + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - res = sym_new_type(ctx, &PyFloat_Type); + + case _COPY_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + bottom = _stack_item_0; + top = PyStackRef_DUP(bottom); + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) { - if (PyJitRef_IsUnique(left)) { - REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE, 0, 0); + case _COPY_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + bottom = _stack_item_1; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = bottom; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (PyJitRef_IsUnique(right)) { - REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE_RIGHT, 0, 0); + + case _COPY_2_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-2]; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - // Result may be a unique compact int or a cached small int - // at runtime. Mark as unique; inplace ops verify at runtime. - res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } - op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { - if (PyJitRef_IsUnique(left)) { - REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE, 0, 0); + case _COPY_2_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + bottom = stack_pointer[-1]; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = _stack_item_0; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (PyJitRef_IsUnique(right)) { - REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, 0, 0); + + case _COPY_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + bottom = _stack_item_0; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = _stack_item_1; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } - op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) { - if (PyJitRef_IsUnique(left)) { - REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE, 0, 0); + case _COPY_3_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-3]; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (PyJitRef_IsUnique(right)) { - REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, 0, 0); + + case _COPY_3_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + bottom = stack_pointer[-2]; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } - op(_BINARY_OP_ADD_FLOAT, (left, right -- res, l, r)) { - if (PyJitRef_IsUnique(left)) { - ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE, 0, 0); - l = PyJitRef_Borrow(sym_new_null(ctx)); - r = right; + case _COPY_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + bottom = stack_pointer[-1]; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (PyJitRef_IsUnique(right)) { - ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, 0, 0); - l = left; - r = PyJitRef_Borrow(sym_new_null(ctx)); + + case _COPY_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + bottom = _stack_item_0; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = _stack_item_2; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[0] = bottom; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - l = left; - r = right; + + case _COPY_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + oparg = CURRENT_OPARG(); + bottom = stack_pointer[-1 - (oparg-1)]; + top = PyStackRef_DUP(bottom); + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); - } - op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res, l, r)) { - if (PyJitRef_IsUnique(left)) { - ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE, 0, 0); - l = PyJitRef_Borrow(sym_new_null(ctx)); - r = right; + case _BINARY_OP_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef rhs; + _PyStackRef lhs; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + rhs = _stack_item_1; + lhs = _stack_item_0; + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + stack_pointer[0] = lhs; + stack_pointer[1] = rhs; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = lhs; + r = rhs; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (PyJitRef_IsUnique(right)) { - ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, 0, 0); - l = left; - r = PyJitRef_Borrow(sym_new_null(ctx)); + + case _SWAP_2_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - l = left; - r = right; + + case _SWAP_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + top = _stack_item_0; + bottom = stack_pointer[-1]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); - } - op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res, l, r)) { - if (PyJitRef_IsUnique(left)) { - ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE, 0, 0); - l = PyJitRef_Borrow(sym_new_null(ctx)); - r = right; + case _SWAP_2_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + top = _stack_item_1; + bottom = _stack_item_0; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (PyJitRef_IsUnique(right)) { - ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, 0, 0); - l = left; - r = PyJitRef_Borrow(sym_new_null(ctx)); + + case _SWAP_2_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + top = _stack_item_2; + bottom = _stack_item_1; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = bottom; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - l = left; - r = right; + + case _SWAP_3_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-3]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_3_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + top = _stack_item_0; + bottom = stack_pointer[-2]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); - } - op(_BINARY_OP_ADD_UNICODE, (left, right -- res, l, r)) { - res = sym_new_type(ctx, &PyUnicode_Type); - l = left; - r = right; - } + case _SWAP_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + top = _stack_item_1; + bottom = stack_pointer[-1]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = _stack_item_0; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + top = _stack_item_2; + bottom = _stack_item_0; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = _stack_item_1; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + top = _stack_item_0; + bottom = stack_pointer[-1 - (oparg-2)]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[-1 - (oparg-2)] = bottom; + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ + + /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */ - op(_BINARY_OP_EXTEND, (descr/4, left, right -- res, l, r)) { - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr *)descr; - if (d != NULL && d->result_type != NULL) { - res = sym_new_type(ctx, d->result_type); - if (d->result_unique) { - res = PyJitRef_MakeUnique(res); + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ + + case _GUARD_IS_TRUE_POP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + flag = stack_pointer[-1]; + int is_true = PyStackRef_IsTrue(flag); + if (!is_true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - res = sym_new_not_null(ctx); + + case _GUARD_IS_TRUE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + flag = _stack_item_0; + int is_true = PyStackRef_IsTrue(flag); + if (!is_true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - l = left; - r = right; - } - op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyUnicode_Type); - } - GETLOCAL(this_instr->operand0) = sym_new_null(ctx); - } - - op(_BINARY_OP_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) { - getitem = sym_new_not_null(ctx); - PyTypeObject *tp = sym_get_type(container); - if (tp == NULL) { - PyObject *c = sym_get_probable_value(container); - if (c != NULL) { - tp = Py_TYPE(c); - } - } - if (tp != NULL) { - PyObject *getitem_o = ((PyHeapTypeObject *)tp)->_spec_cache.getitem; - sym_set_recorded_value(getitem, getitem_o); - } - } - - op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { - _Py_UOpsAbstractFrame *f = frame_new_from_symbol(ctx, getitem, NULL, 0); - if (f == NULL) { - break; - } - f->locals[0] = container; - f->locals[1] = sub; - new_frame = PyJitRef_WrapInvalid(f); - } - - op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res, s, i)) { - res = sym_new_type(ctx, &PyUnicode_Type); - s = str_st; - i = sub_st; - } - - op(_BINARY_OP_SUBSCR_USTR_INT, (str_st, sub_st -- res, s, i)) { - res = sym_new_type(ctx, &PyUnicode_Type); - s = str_st; - i = sub_st; - } - - op(_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, (tuple_st, sub_st -- tuple_st, sub_st)) { - assert(sym_matches_type(tuple_st, &PyTuple_Type)); - if (sym_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); - long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); - assert(index >= 0); - Py_ssize_t tuple_length = sym_tuple_length(tuple_st); - if (tuple_length != -1 && index < tuple_length) { - ADD_OP(_NOP, 0, 0); - } - } - } - - op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res, ts, ss)) { - assert(sym_matches_type(tuple_st, &PyTuple_Type)); - if (sym_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); - long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); - assert(index >= 0); - Py_ssize_t tuple_length = sym_tuple_length(tuple_st); - if (tuple_length == -1) { - // Unknown length - res = sym_new_not_null(ctx); + case _GUARD_IS_TRUE_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + flag = _stack_item_1; + int is_true = PyStackRef_IsTrue(flag); + if (!is_true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - else { - assert(index < tuple_length); - res = sym_tuple_getitem(ctx, tuple_st, index); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_IS_TRUE_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + flag = _stack_item_2; + int is_true = PyStackRef_IsTrue(flag); + if (!is_true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - } - else { - res = sym_new_not_null(ctx); - } - ts = tuple_st; - ss = sub_st; - } - - op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res, ds, ss)) { - res = sym_new_not_null(ctx); - ds = dict_st; - ss = sub_st; - PyObject *sub = sym_get_const(ctx, sub_st); - if (sym_is_not_container(sub_st) && - sym_matches_type(dict_st, &PyFrozenDict_Type)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(dict_st, sub_st, res); - } - else if (sub != NULL) { - optimize_dict_known_hash(ctx, dependencies, this_instr, - sub, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH); - } - } - - op(_BINARY_OP_SUBSCR_LIST_SLICE, (list_st, sub_st -- res, ls, ss)) { - res = sym_new_type(ctx, &PyList_Type); - ls = list_st; - ss = sub_st; - } - - op(_TO_BOOL, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, - _POP_TOP, _NOP); - if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); - } - } - - op(_TO_BOOL_BOOL, (value -- value)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &value, - _POP_TOP, _NOP); - if (!already_bool) { - sym_set_type(value, &PyBool_Type); - } - } - - op(_TO_BOOL_INT, (value -- res, v)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, - _NOP, _SWAP); - if (!already_bool) { - sym_set_type(value, &PyLong_Type); - res = sym_new_truthiness(ctx, value, true); - } - v = value; - } - - op(_TO_BOOL_LIST, (value -- res, v)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, - _NOP, _SWAP); - if (!already_bool) { - res = sym_new_type(ctx, &PyBool_Type); - } - v = value; - } - - op(_TO_BOOL_NONE, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, - _POP_TOP, _NOP); - if (!already_bool) { - sym_set_const(value, Py_None); - res = sym_new_const(ctx, Py_False); - } - } - - op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyUnicode_Type)) { - ADD_OP(_NOP, 0, 0); - } - sym_set_type(nos, &PyUnicode_Type); - } - - op(_GUARD_NOS_COMPACT_ASCII, (nos, unused -- nos, unused)) { - sym_set_type(nos, &PyUnicode_Type); - } - - op(_GUARD_TOS_UNICODE, (value -- value)) { - if (sym_matches_type(value, &PyUnicode_Type)) { - ADD_OP(_NOP, 0, 0); - } - sym_set_type(value, &PyUnicode_Type); - } - - op(_TO_BOOL_STR, (value -- res, v)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, - _NOP, _SWAP); - v = value; - if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); - } - } - - op(_UNARY_NOT, (value -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(value, res); - sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, false); - } - - op(_UNARY_NEGATIVE, (value -- res, v)) { - v = value; - REPLACE_OPCODE_IF_EVALUATES_PURE(value, res); - if (sym_matches_type(value, &PyFloat_Type) && PyJitRef_IsUnique(value)) { - ADD_OP(_UNARY_NEGATIVE_FLOAT_INPLACE, 0, 0); - v = PyJitRef_Borrow(sym_new_null(ctx)); - res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); - } - else if (sym_is_compact_int(value)) { - res = sym_new_compact_int(ctx); - } - else { - PyTypeObject *type = sym_get_type(value); - if (type == &PyLong_Type || type == &PyFloat_Type) { - res = sym_new_type(ctx, type); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_IS_FALSE_POP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + flag = stack_pointer[-1]; + int is_false = PyStackRef_IsFalse(flag); + if (!is_false) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); } - else { - res = sym_new_not_null(ctx); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_IS_FALSE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + flag = _stack_item_0; + int is_false = PyStackRef_IsFalse(flag); + if (!is_false) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_UNARY_INVERT, (value -- res, v)) { - v = value; - // Required to avoid a warning due to the deprecation of bitwise inversion of bools - if (!sym_matches_type(value, &PyBool_Type)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(value, res); + case _GUARD_IS_FALSE_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + flag = _stack_item_1; + int is_false = PyStackRef_IsFalse(flag); + if (!is_false) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - if (sym_matches_type(value, &PyLong_Type)) { - res = sym_new_type(ctx, &PyLong_Type); + + case _GUARD_IS_FALSE_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + flag = _stack_item_2; + int is_false = PyStackRef_IsFalse(flag); + if (!is_false) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - res = sym_new_not_null(ctx); + + case _GUARD_BIT_IS_SET_POP_4_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_COMPARE_OP, (left, right -- res)) { - // Comparison between bytes and str or int is not impacted by this optimization as bytes - // is not a safe type (due to its ability to raise a warning during comparisons). - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - if (oparg & 16) { - res = sym_new_type(ctx, &PyBool_Type); - } - else { - res = _Py_uop_sym_new_not_null(ctx); - } - } - - op(_COMPARE_OP_INT, (left, right -- res, l, r)) { - int cmp_mask = oparg & (COMPARE_LT_MASK | COMPARE_GT_MASK | COMPARE_EQ_MASK); - - if (cmp_mask == COMPARE_EQ_MASK) { - res = sym_new_predicate(ctx, left, right, JIT_PRED_EQ); - } - else if (cmp_mask == (COMPARE_LT_MASK | COMPARE_GT_MASK)) { - res = sym_new_predicate(ctx, left, right, JIT_PRED_NE); - } - else { - res = sym_new_type(ctx, &PyBool_Type); - } - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } - - op(_COMPARE_OP_FLOAT, (left, right -- res, l, r)) { - int cmp_mask = oparg & (COMPARE_LT_MASK | COMPARE_GT_MASK | COMPARE_EQ_MASK); - - if (cmp_mask == COMPARE_EQ_MASK) { - res = sym_new_predicate(ctx, left, right, JIT_PRED_EQ); - } - else if (cmp_mask == (COMPARE_LT_MASK | COMPARE_GT_MASK)) { - res = sym_new_predicate(ctx, left, right, JIT_PRED_NE); - } - else { - res = sym_new_type(ctx, &PyBool_Type); - } - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } - - op(_COMPARE_OP_STR, (left, right -- res, l, r)) { - /* Cannot use predicate optimization here, as `a == b` - * does not imply that `a` is equivalent to `b`. `a` may be - * mortal, while `b` is immortal */ - res = sym_new_type(ctx, &PyBool_Type); - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } + case _GUARD_BIT_IS_SET_POP_4_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_IS_OP, (left, right -- b, l, r)) { - b = sym_new_predicate(ctx, left, right, (oparg ? JIT_PRED_IS_NOT : JIT_PRED_IS)); - l = left; - r = right; - } - - op(_IS_NONE, (value -- b)) { - if (sym_is_const(ctx, value)) { - PyObject *value_o = sym_get_const(ctx, value); - b = sym_new_const(ctx, Py_IsNone(value_o) ? Py_True : Py_False); - } - else { - b = sym_new_type(ctx, &PyBool_Type); - } - } - - op(_CONTAINS_OP, (left, right -- b, l, r)) { - b = sym_new_type(ctx, &PyBool_Type); - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, b); - } - - op(_CONTAINS_OP_SET, (left, right -- b, l, r)) { - b = sym_new_type(ctx, &PyBool_Type); - l = left; - r = right; - if (sym_is_not_container(left) && - sym_matches_type(right, &PyFrozenSet_Type)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, b); - } - } - - op(_CONTAINS_OP_DICT, (left, right -- b, l, r)) { - b = sym_new_type(ctx, &PyBool_Type); - l = left; - r = right; - } - - op(_LOAD_CONST, (-- value)) { - PyCodeObject *co = get_current_code_object(ctx); - PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - - op(_LOAD_COMMON_CONSTANT, (-- value)) { - assert(oparg < NUM_COMMON_CONSTANTS); - PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - - op(_LOAD_SMALL_INT, (-- value)) { - PyObject *val = PyLong_FromLong(oparg); - assert(val); - assert(_Py_IsImmortal(val)); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - - op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); - } - - op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); - } - - op(_POP_TOP_OPARG, (args[oparg] --)) { - for (int i = oparg-1; i >= 0; i--) { - optimize_pop_top(ctx, this_instr, args[i]); - } - } - - op(_POP_TOP, (value -- )) { - optimize_pop_top(ctx, this_instr, value); - } - - op(_POP_TOP_INT, (value --)) { - if (PyJitRef_IsBorrowed(value)) { - ADD_OP(_POP_TOP_NOP, 0, 0); - } - } - - op(_POP_TOP_FLOAT, (value --)) { - if (PyJitRef_IsBorrowed(value)) { - ADD_OP(_POP_TOP_NOP, 0, 0); - } - } - - op(_POP_TOP_UNICODE, (value --)) { - if (PyJitRef_IsBorrowed(value)) { - ADD_OP(_POP_TOP_NOP, 0, 0); - } - } - - op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { - assert(oparg > 0); - bottom = PyJitRef_RemoveUnique(bottom); - top = bottom; - } - - op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { - JitOptRef temp = bottom; - bottom = top; - top = temp; - assert(oparg >= 2); - } - - op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, o)) { - attr = sym_new_not_null(ctx); - (void)offset; - o = owner; - } - - op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr, o)) { - (void)dict_version; - (void)index; - attr = PyJitRef_NULL; - if (sym_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); - if (PyModule_CheckExact(mod)) { - PyObject *dict = mod->md_dict; - uint64_t watched_mutations = get_mutations(dict); - if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - PyDict_Watch(GLOBALS_WATCHER_ID, dict); - _Py_BloomFilter_Add(dependencies, dict); - PyObject *res = convert_global_to_const(this_instr, dict); - if (res == NULL) { - attr = sym_new_not_null(ctx); - } - else { - bool immortal = _Py_IsImmortal(res); - ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, - 0, (uintptr_t)res); - ADD_OP(_SWAP, 2, 0); - attr = sym_new_const(ctx, res); - } + case _GUARD_BIT_IS_SET_POP_4_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - } + case _GUARD_BIT_IS_SET_POP_4_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - if (PyJitRef_IsNull(attr)) { - /* No conversion made. We don't know what `attr` is. */ - attr = sym_new_not_null(ctx); - } - o = owner; - } - - op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { - if (oparg & 1) { - ADD_OP(_PUSH_NULL, 0, 0); - null[0] = sym_new_null(ctx); - } - else { - ADD_OP(_NOP, 0, 0); - } - } - - op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { - (void)owner; - attr = sym_new_not_null(ctx); - if (oparg & 1) { - self_or_null[0] = sym_new_unknown(ctx); - } - } - - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, o)) { - attr = sym_new_not_null(ctx); - (void)hint; - o = owner; - } - - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, o)) { - attr = sym_new_not_null(ctx); - (void)index; - o = owner; - } - - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { - (void)descr; - PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); - PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, - _POP_TOP, _NOP); - } - - op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { - (void)descr; - PyTypeObject *type = sym_get_type(owner); - PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, - _POP_TOP, _NOP); - } - - op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { - (void)descr; - PyTypeObject *type = sym_get_type(owner); - PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, - _POP_TOP, _NOP); - } - - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { - (void)descr; - PyTypeObject *type = sym_get_type(owner); - PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, - _NOP, _SWAP); - self = owner; - } - - op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { - (void)descr; - PyTypeObject *type = sym_get_type(owner); - PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, - _NOP, _SWAP); - self = owner; - } - - op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { - (void)descr; - PyTypeObject *type = sym_get_type(owner); - PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, - _NOP, _SWAP); - self = owner; - } - - op(_GUARD_LOAD_SUPER_ATTR_METHOD, (global_super_st, class_st, unused -- global_super_st, class_st, unused)) { - if (sym_get_const(ctx, global_super_st) == (PyObject *)&PySuper_Type) { - PyTypeObject *probable = (PyTypeObject *)sym_get_probable_value(class_st); - PyTypeObject *known = (PyTypeObject *)sym_get_const(ctx, class_st); - // not known, but has a probable type, promote the probable type - if (known == NULL && probable != NULL && PyType_Check(probable)) { - ADD_OP(_GUARD_NOS_TYPE_VERSION, 0, probable->tp_version_tag); - known = probable; - } - sym_set_const(class_st, (PyObject *)known); - } - else { - sym_set_const(global_super_st, (PyObject *)&PySuper_Type); - sym_set_type(class_st, &PyType_Type); - } - } - - op(_LOAD_SUPER_ATTR_METHOD, (global_super_st, class_st, self_st -- attr, self_or_null)) { - self_or_null = self_st; - PyTypeObject *su_type = (PyTypeObject *)sym_get_const(ctx, class_st); - PyTypeObject *obj_type = sym_get_type(self_st); - PyObject *name = get_co_name(ctx, oparg >> 2); - attr = lookup_super_attr(ctx, dependencies, this_instr, - su_type, obj_type, name, - _LOAD_CONST_INLINE_BORROW, - _LOAD_CONST_INLINE, _SWAP); - } - - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { - // + 1 for _SAVE_RETURN_OFFSET - // FIX ME -- This needs a version check and function watcher - PyCodeObject *co = (PyCodeObject *)((PyFunctionObject *)fget)->func_code; - _Py_UOpsAbstractFrame *f = frame_new(ctx, co, NULL, 0); - if (f == NULL) { - break; - } - f->locals[0] = owner; - new_frame = PyJitRef_WrapInvalid(f); - } - - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); - } - - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (sym_get_func_version(callable) == func_version) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - sym_set_func_version(ctx, callable, func_version); - } - } - - op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { - if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { - PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); - assert(PyMethod_Check(method)); - ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - uop_buffer_last(&ctx->out_buffer)->operand1 = (uintptr_t)method->im_func; - } - sym_set_type(callable, &PyMethod_Type); - } - - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(ctx, callable)) { - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); - PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { - ADD_OP(_NOP, 0 ,0); - } - } - } - } - - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); - } - - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { - int argcount = oparg; - assert(!PyJitRef_IsNull(self_or_null)); - assert(args != NULL); - if (sym_is_not_null(self_or_null)) { - // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM - args--; - argcount++; - } - - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, args, argcount)); - } else { - new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); - } - } - - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); - } - - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { - new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); - } - - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); - } - - op(_PY_FRAME_EX, (func_st, null, callargs_st, kwargs_st -- ex_frame)) { - ex_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, func_st, NULL, 0)); - } - - op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - (void)args; - PyObject *probable_callable = sym_get_probable_value(callable); - assert(probable_callable != NULL); - assert(PyType_Check(probable_callable)); - PyTypeObject *tp = (PyTypeObject *)probable_callable; - if (tp->tp_version_tag == type_version) { - // If the type version has not changed since we last saw it, - // then we know this __init__ is definitely the same one as in the cache. - // We can promote callable to a known constant. This does not need a - // type watcher, as we do not remove this _CHECK_AND_ALLOCATE_OBJECT guard. - // TODO: split up _CHECK_AND_ALLOCATE_OBJECT to the check then alloate, so we can - // eliminate the check. - PyHeapTypeObject *cls = (PyHeapTypeObject *)probable_callable; - PyObject *init = cls->_spec_cache.init; - assert(init != NULL); - assert(PyFunction_Check(init)); - callable = sym_new_const(ctx, init); - } - else { - callable = sym_new_not_null(ctx); - } - self_or_null = sym_new_not_null(ctx); - } - - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - ctx->frame->stack_pointer = stack_pointer - oparg - 2; - _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, NULL, 0); - if (shim == NULL) { - break; - } - /* Push self onto stack of shim */ - shim->stack_pointer[0] = self; - shim->stack_pointer++; - assert((int)(shim->stack_pointer - shim->stack) == 1); - ctx->frame = shim; - ctx->curr_frame_depth++; - assert((this_instr + 1)->opcode == _PUSH_FRAME); - init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, oparg+1)); - } - - op(_RETURN_VALUE, (retval -- res)) { - JitOptRef temp = retval; - DEAD(retval); - SAVE_STACK(); - ctx->frame->stack_pointer = stack_pointer; - assert(this_instr[1].opcode == _RECORD_CODE); - PyCodeObject *returning_code = (PyCodeObject *)this_instr[1].operand0; - assert(PyCode_Check(returning_code)); - if (returning_code == NULL) { - ctx->done = true; - break; - } - if (frame_pop(ctx, returning_code)) { - break; - } - stack_pointer = ctx->frame->stack_pointer; - - RELOAD_STACK(); - res = temp; - } - - op(_RETURN_GENERATOR, ( -- res)) { - SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; - assert(this_instr[1].opcode == _RECORD_CODE); - PyCodeObject *returning_code = (PyCodeObject *)this_instr[1].operand0; - if (returning_code == NULL) { - ctx->done = true; - break; - } - assert(PyCode_Check(returning_code)); - if (frame_pop(ctx, returning_code)) { - break; - } - stack_pointer = ctx->frame->stack_pointer; - res = sym_new_unknown(ctx); - } - - op(_YIELD_VALUE, (retval -- value)) { - JitOptRef temp = retval; - DEAD(retval); - SAVE_STACK(); - ctx->frame->stack_pointer = stack_pointer; - assert(this_instr[1].opcode == _RECORD_CODE); - PyCodeObject *returning_code = (PyCodeObject *)this_instr[1].operand0; - if (returning_code == NULL) { - ctx->done = true; - break; - } - assert(PyCode_Check(returning_code)); - if (frame_pop(ctx, returning_code)) { - break; - } - stack_pointer = ctx->frame->stack_pointer; - RELOAD_STACK(); - value = temp; - } - op(_GET_ITER, (iterable -- iter, index_or_null)) { - if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { - iter = iterable; - index_or_null = sym_new_not_null(ctx); + case _GUARD_BIT_IS_SET_POP_5_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - iter = sym_new_not_null(ctx); - index_or_null = sym_new_unknown(ctx); + + case _GUARD_BIT_IS_SET_POP_5_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_FOR_ITER_GEN_FRAME, (iter, unused -- iter, unused, gen_frame)) { - _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, iter, NULL, 0); - if (new_frame == NULL) { - ctx->done = true; + case _GUARD_BIT_IS_SET_POP_5_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - new_frame->stack_pointer[0] = sym_new_const(ctx, Py_None); - new_frame->stack_pointer++; - gen_frame = PyJitRef_WrapInvalid(new_frame); - } - op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { - _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, receiver, NULL, 0); - if (new_frame == NULL) { - ctx->done = true; + case _GUARD_BIT_IS_SET_POP_5_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - new_frame->stack_pointer[0] = PyJitRef_StripReferenceInfo(v); - new_frame->stack_pointer++; - gen_frame = PyJitRef_WrapInvalid(new_frame); - } - op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - PyCodeObject *co = sym_get_probable_func_code(callable); - if (co == NULL) { - ctx->done = true; + case _GUARD_BIT_IS_SET_POP_6_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - ADD_OP(_CHECK_STACK_SPACE_OPERAND, 0, co->co_framesize); - } - op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { - (void)framesize; - } + case _GUARD_BIT_IS_SET_POP_6_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_PUSH_FRAME, (new_frame -- )) { - SYNC_SP(); - if (!CURRENT_FRAME_IS_INIT_SHIM()) { - ctx->frame->stack_pointer = stack_pointer; + case _GUARD_BIT_IS_SET_POP_6_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - ctx->frame->caller = true; - ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); - ctx->curr_frame_depth++; - stack_pointer = ctx->frame->stack_pointer; - assert(ctx->frame->locals != NULL); - } - op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { - (void)top; - /* This has to be done manually */ - for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); - } - } - - op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8], top[0])) { - (void)top; - /* This has to be done manually */ - int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; - for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); - } - } - - op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - if (sym_matches_type(iter, &PyTuple_Type)) { - ADD_OP(_NOP, 0, 0); - } - sym_set_type(iter, &PyTuple_Type); - } - - op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { - if (sym_matches_type(iter, &PyList_Type)) { - ADD_OP(_NOP, 0, 0); - } - else { - sym_set_type(iter, &PyList_Type); + case _GUARD_BIT_IS_SET_POP_6_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - - op(_ITER_CHECK_RANGE, (iter, null_or_index -- iter, null_or_index)) { - if (sym_matches_type(iter, &PyRange_Type)) { - ADD_OP(_NOP, 0, 0); - } - else { - sym_set_type(iter, &PyRange_Type); - } - } - - op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { - next = sym_new_type(ctx, &PyLong_Type); - } - - op(_CALL_TYPE_1, (unused, unused, arg -- res, a)) { - PyObject* type = (PyObject *)sym_get_type(arg); - if (type) { - res = sym_new_const(ctx, type); - ADD_OP(_SWAP, 3, 0); - ADD_OP(_POP_TOP, 0, 0); - ADD_OP(_POP_TOP, 0, 0); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)type); - ADD_OP(_SWAP, 2, 0); - } - else { - res = sym_new_not_null(ctx); - } - a = arg; - } - - op(_CALL_STR_1, (unused, unused, arg -- res, a)) { - if (sym_matches_type(arg, &PyUnicode_Type)) { - // e.g. str('foo') or str(foo) where foo is known to be a string - // Note: we must strip the reference information because it goes - // through str() which strips the reference information from it. - res = PyJitRef_StripReferenceInfo(arg); - } - else { - res = sym_new_type(ctx, &PyUnicode_Type); - } - a = arg; - } - - op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { - // the result is always a bool, but sometimes we can - // narrow it down to True or False - res = sym_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = sym_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); - if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { - // isinstance(inst, cls) where both inst and cls have - // known types, meaning we can deduce either True or False - - // The below check is equivalent to PyObject_TypeCheck(inst, cls) - PyObject *out = Py_False; - if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { - out = Py_True; - } - sym_set_const(res, out); - ADD_OP(_POP_TOP, 0, 0); - ADD_OP(_POP_TOP, 0, 0); - ADD_OP(_POP_TOP_NOP, 0, 0); - ADD_OP(_POP_TOP, 0, 0); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); - } - } - - op(_CALL_LIST_APPEND, (callable, self, arg -- none, c, s)) { - (void)(arg); - c = callable; - s = self; - none = sym_new_const(ctx, Py_None); - } - - op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyType_Type)) { - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (tp->tp_vectorcall != NULL) { - ADD_OP(_NOP, 0, 0); + + case _GUARD_BIT_IS_SET_POP_7_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyType_Type); + + case _GUARD_BIT_IS_SET_POP_7_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && - (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; - if (sym_is_not_null(self_or_null)) { - total_args++; + case _GUARD_BIT_IS_SET_POP_7_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { - ADD_OP(_NOP, 0, 0); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_SET_POP_7_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyCFunction_Type); + + case _GUARD_BIT_IS_SET_POP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = CURRENT_OPARG(); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { - ADD_OP(_NOP, 0, 0); + case _GUARD_BIT_IS_SET_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyCFunction_Type); + + case _GUARD_BIT_IS_SET_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { - ADD_OP(_NOP, 0, 0); + case _GUARD_BIT_IS_SET_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyCFunction_Type); + + case _GUARD_BIT_IS_UNSET_POP_4_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - callable = sym_new_not_null(ctx); - } + case _GUARD_BIT_IS_UNSET_POP_4_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { - res = sym_new_not_null(ctx); - c = callable; - if (sym_is_not_null(self_or_null)) { - args--; - s = args[0]; + case _GUARD_BIT_IS_UNSET_POP_4_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (sym_is_null(self_or_null)) { - s = args[0]; + + case _GUARD_BIT_IS_UNSET_POP_4_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - s = sym_new_unknown(ctx); + + case _GUARD_BIT_IS_UNSET_POP_5_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - callable = sym_new_not_null(ctx); - } + case _GUARD_BIT_IS_UNSET_POP_5_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - callable = sym_new_not_null(ctx); - } + case _GUARD_BIT_IS_UNSET_POP_5_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && - (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; - if (sym_is_not_null(self_or_null)) { - total_args++; + case _GUARD_BIT_IS_UNSET_POP_5_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - PyTypeObject *self_type = NULL; - if (sym_is_not_null(self_or_null)) { - self_type = sym_get_type(self_or_null); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_6_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); } - else { - self_type = sym_get_type(args[0]); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_6_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; - if (total_args == 2 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && - self_type == d_type) { - ADD_OP(_NOP, 0, 0); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_6_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyMethodDescr_Type); + + case _GUARD_BIT_IS_UNSET_POP_6_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && - (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; - if (sym_is_not_null(self_or_null)) { - total_args++; + case _GUARD_BIT_IS_UNSET_POP_7_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); } - PyTypeObject *self_type = NULL; - if (sym_is_not_null(self_or_null)) { - self_type = sym_get_type(self_or_null); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_7_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - else { - self_type = sym_get_type(args[0]); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_7_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; - if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && - self_type == d_type) { - ADD_OP(_NOP, 0, 0); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_7_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyMethodDescr_Type); + + case _GUARD_BIT_IS_UNSET_POP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + oparg = CURRENT_OPARG(); + flag = stack_pointer[-1]; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && - (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; - if (sym_is_not_null(self_or_null)) { - total_args++; + case _GUARD_BIT_IS_UNSET_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + flag = _stack_item_0; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - PyTypeObject *self_type = NULL; - if (sym_is_not_null(self_or_null)) { - self_type = sym_get_type(self_or_null); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + flag = _stack_item_1; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - else { - self_type = sym_get_type(args[0]); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BIT_IS_UNSET_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + flag = _stack_item_2; + #ifdef Py_STACKREF_DEBUG + uintptr_t bits = flag.index; + #else + uintptr_t bits = flag.bits; + #endif + uintptr_t set = (1 << oparg) & bits; + if (set != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_IS_NONE_POP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef val; + val = stack_pointer[-1]; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; - if (total_args == 1 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && - self_type == d_type) { - ADD_OP(_NOP, 0, 0); + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_IS_NONE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + val = _stack_item_0; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyMethodDescr_Type); + + case _GUARD_IS_NONE_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + val = _stack_item_1; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CHECK_RECURSION_LIMIT, ( -- )) { - if (ctx->frame->is_c_recursion_checked) { - ADD_OP(_NOP, 0, 0); + case _GUARD_IS_NONE_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + val = _stack_item_2; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - ctx->frame->is_c_recursion_checked = true; - } - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res, c, s)) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) - && sym_is_not_null(self_or_null)) { - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); + case _GUARD_IS_NOT_NONE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + val = _stack_item_0; + int is_none = PyStackRef_IsNone(val); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_none) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - res = sym_new_not_null(ctx); - c = callable; - if (sym_is_not_null(self_or_null)) { - args--; - s = args[0]; + + case _JUMP_TO_TOP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + JUMP_TO_JUMP_TARGET(); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (sym_is_null(self_or_null)) { - s = args[0]; + + case _SET_IP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - s = sym_new_unknown(ctx); + + case _SET_IP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) - && sym_is_not_null(self_or_null)) { - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); + case _SET_IP_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - callable = sym_new_not_null(ctx); - } - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) - && sym_is_not_null(self_or_null)) { - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); + case _SET_IP_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - callable = sym_new_not_null(ctx); - } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && - (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; - if (sym_is_not_null(self_or_null)) { - total_args++; + case _CHECK_STACK_SPACE_OPERAND_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - PyTypeObject *self_type = NULL; - if (sym_is_not_null(self_or_null)) { - self_type = sym_get_type(self_or_null); - } - else { - self_type = sym_get_type(args[0]); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_STACK_SPACE_OPERAND_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; - if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && - self_type == d_type) { - ADD_OP(_NOP, 0, 0); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _CHECK_STACK_SPACE_OPERAND_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(callable, &PyMethodDescr_Type); + + case _CHECK_STACK_SPACE_OPERAND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { - PyObject *callable_o = sym_get_const(ctx, callable); - if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) - && sym_is_not_null(self_or_null)) { - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); + case _SAVE_RETURN_OFFSET_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - res = sym_new_not_null(ctx); - c = callable; - if (sym_is_not_null(self_or_null)) { - args--; - s = args[0]; - a = args[1]; + + case _SAVE_RETURN_OFFSET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - s = sym_new_unknown(ctx); - a = sym_new_unknown(ctx); + + case _SAVE_RETURN_OFFSET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CALL_INTRINSIC_1, (value -- res, v)) { - res = sym_new_not_null(ctx); - v = value; - } - - op(_CALL_INTRINSIC_2, (value2_st, value1_st -- res, vs1, vs2)) { - res = sym_new_not_null(ctx); - vs1 = value1_st; - vs2 = value2_st; - } - - op(_GUARD_IS_TRUE_POP, (flag -- )) { - sym_apply_predicate_narrowing(ctx, flag, true); - - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); - assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, value != Py_True); - } - else { - int bit = get_test_bit_for_bools(); - if (bit) { - REPLACE_OP(this_instr, - test_bit_set_in_true(bit) ? - _GUARD_BIT_IS_SET_POP : - _GUARD_BIT_IS_UNSET_POP, bit, 0); - } - } - sym_set_const(flag, Py_True); - } - - op(_GUARD_IS_FALSE_POP, (flag -- )) { - sym_apply_predicate_narrowing(ctx, flag, false); - - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); - assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, value != Py_False); - } - else { - int bit = get_test_bit_for_bools(); - if (bit) { - REPLACE_OP(this_instr, - test_bit_set_in_true(bit) ? - _GUARD_BIT_IS_UNSET_POP : - _GUARD_BIT_IS_SET_POP, bit, 0); - } - } - sym_set_const(flag, Py_False); - } - - op(_GUARD_IS_NONE_POP, (val -- )) { - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); - assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); - } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, true); - } - sym_set_const(val, Py_None); - } - - op(_GUARD_IS_NOT_NONE_POP, (val -- )) { - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); - assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); - } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, false); - } - } - - op(_CHECK_PEP_523, (--)) { - /* Setting the eval frame function invalidates - * all executors, so no need to check dynamically */ - if (_PyInterpreterState_GET()->eval_frame == NULL) { - ADD_OP(_NOP, 0 ,0); - } - } - - op(_INSERT_NULL, (self -- method_and_self[2])) { - method_and_self[0] = sym_new_null(ctx); - method_and_self[1] = self; - } - - op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { - method_and_self[0] = sym_new_not_null(ctx); - method_and_self[1] = sym_new_unknown(ctx); - } - - op(_JUMP_TO_TOP, (--)) { - ctx->done = true; - } - - op(_EXIT_TRACE, (exit_p/4 --)) { - (void)exit_p; - ctx->done = true; - } - - op(_DEOPT, (--)) { - ctx->done = true; - } - - op(_REPLACE_WITH_TRUE, (value -- res, v)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - ADD_OP(_SWAP, 2, 0); - res = sym_new_const(ctx, Py_True); - v = value; - } + case _SAVE_RETURN_OFFSET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_BUILD_TUPLE, (values[oparg] -- tup)) { - tup = sym_new_tuple(ctx, oparg, values); - tup = PyJitRef_MakeUnique(tup); - } + case _EXIT_TRACE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + TIER2_TO_TIER2(exit->executor); + } - op(_BUILD_LIST, (values[oparg] -- list)) { - list = sym_new_type(ctx, &PyList_Type); - } + case _EXIT_TRACE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + } + #endif + tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + TIER2_TO_TIER2(exit->executor); + } - op(_BUILD_SLICE, (args[oparg] -- slice)) { - slice = sym_new_type(ctx, &PySlice_Type); - } + case _EXIT_TRACE_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + } + #endif + tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + TIER2_TO_TIER2(exit->executor); + } - op(_BUILD_MAP, (values[oparg*2] -- map)) { - map = sym_new_type(ctx, &PyDict_Type); - } + case _EXIT_TRACE_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + } + #endif + tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + TIER2_TO_TIER2(exit->executor); + } - op(_BUILD_STRING, (pieces[oparg] -- str)) { - str = sym_new_type(ctx, &PyUnicode_Type); - } + case _DYNAMIC_EXIT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(frame->instr_ptr); + } - op(_BUILD_SET, (values[oparg] -- set)) { - set = sym_new_type(ctx, &PySet_Type); - } + case _DYNAMIC_EXIT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + } + #endif + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + GOTO_TIER_ONE(frame->instr_ptr); + } - op(_SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1], i)) { - (void)set; - i = iterable; - } + case _DYNAMIC_EXIT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + } + #endif + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + GOTO_TIER_ONE(frame->instr_ptr); + } - op(_LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1], i)) { - (void)list_st; - i = iterable_st; - } + case _DYNAMIC_EXIT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + } + #endif + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + GOTO_TIER_ONE(frame->instr_ptr); + } - op(_DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1], u)) { - (void)callable; - (void)dict; - u = update; - } + case _CHECK_VALIDITY_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { - if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 2) { - ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, oparg, 0); + case _CHECK_VALIDITY_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - val0 = sym_tuple_getitem(ctx, seq, 0); - val1 = sym_tuple_getitem(ctx, seq, 1); - } - op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { - if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 3) { - ADD_OP(_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, oparg, 0); + case _CHECK_VALIDITY_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == oparg) { - ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TUPLE, oparg, 0); + + case _CHECK_VALIDITY_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - for (int i = 0; i < oparg; i++) { - values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); + + case _LOAD_CONST_INLINE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CALL_TUPLE_1, (callable, null, arg -- res, a)) { - if (sym_matches_type(arg, &PyTuple_Type)) { - // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple - // Note: we must strip the reference information because it goes - // through tuple() which strips the reference information from it. - res = PyJitRef_StripReferenceInfo(arg); + case _LOAD_CONST_INLINE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - res = sym_new_type(ctx, &PyTuple_Type); + + case _LOAD_CONST_INLINE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - a = arg; - } - op(_GUARD_TOS_LIST, (tos -- tos)) { - if (sym_matches_type(tos, &PyList_Type)) { - ADD_OP(_NOP, 0, 0); + case _LOAD_CONST_INLINE_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(tos, &PyList_Type); + + case _LOAD_CONST_INLINE_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyList_Type)) { - ADD_OP(_NOP, 0, 0); + case _LOAD_CONST_INLINE_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(nos, &PyList_Type); + + case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + arg = stack_pointer[-1]; + callable = stack_pointer[-3]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + res = PyStackRef_FromPyObjectBorrow(ptr); + a = arg; + c = callable; + _tos_cache2 = c; + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_TOS_TUPLE, (tos -- tos)) { - if (sym_matches_type(tos, &PyTuple_Type)) { - ADD_OP(_NOP, 0, 0); + case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef _stack_item_0 = _tos_cache0; + arg = _stack_item_0; + callable = stack_pointer[-2]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + res = PyStackRef_FromPyObjectBorrow(ptr); + a = arg; + c = callable; + _tos_cache2 = c; + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(tos, &PyTuple_Type); + + case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + arg = _stack_item_1; + callable = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + res = PyStackRef_FromPyObjectBorrow(ptr); + a = arg; + c = callable; + _tos_cache2 = c; + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyTuple_Type)) { - ADD_OP(_NOP, 0, 0); + case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef arg; + _PyStackRef callable; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + arg = _stack_item_2; + callable = _stack_item_0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); + res = PyStackRef_FromPyObjectBorrow(ptr); + a = arg; + c = callable; + _tos_cache2 = c; + _tos_cache1 = a; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(nos, &PyTuple_Type); + + case _START_EXECUTOR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *executor = (PyObject *)CURRENT_OPERAND0_64(); + #ifndef _Py_JIT + assert(current_executor == (_PyExecutorObject*)executor); + #endif + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)current_executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutor_ClearExit(tstate->jit_exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyDict_Type)) { - ADD_OP(_NOP, 0, 0); + case _MAKE_WARM_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + current_executor->vm_data.cold = false; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - sym_set_type(nos, &PyDict_Type); - } - op(_GUARD_NOS_ANY_DICT, (nos, unused -- nos, unused)) { - PyTypeObject *tp = sym_get_type(nos); - if (tp == &PyDict_Type || tp == &PyFrozenDict_Type) { - ADD_OP(_NOP, 0, 0); + case _MAKE_WARM_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + current_executor->vm_data.cold = false; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_TOS_ANY_DICT, (tos -- tos)) { - PyTypeObject *tp = sym_get_type(tos); - if (tp == &PyDict_Type || tp == &PyFrozenDict_Type) { - ADD_OP(_NOP, 0, 0); + case _MAKE_WARM_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + current_executor->vm_data.cold = false; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - // Narrowing the guard based on the probable type. - tp = sym_get_probable_type(tos); - if (tp == &PyDict_Type) { - ADD_OP(_GUARD_TOS_DICT, 0, 0); - } - else if (tp == &PyFrozenDict_Type) { - ADD_OP(_GUARD_TOS_FROZENDICT, 0, 0); - } + + case _MAKE_WARM_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + current_executor->vm_data.cold = false; + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_TOS_DICT, (tos -- tos)) { - if (sym_matches_type(tos, &PyDict_Type)) { - ADD_OP(_NOP, 0, 0); + case _FATAL_ERROR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + assert(0); + Py_FatalError("Fatal error uop executed."); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(tos, &PyDict_Type); + + case _FATAL_ERROR_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_TOS_FROZENDICT, (tos -- tos)) { - if (sym_matches_type(tos, &PyFrozenDict_Type)) { - ADD_OP(_NOP, 0, 0); + case _FATAL_ERROR_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_type(tos, &PyFrozenDict_Type); + + case _FATAL_ERROR_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_TOS_ANY_SET, (tos -- tos)) { - PyTypeObject *tp = sym_get_type(tos); - if (tp == &PySet_Type || tp == &PyFrozenSet_Type) { - ADD_OP(_NOP, 0, 0); + case _DEOPT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - else { - // Narrowing the guard based on the probable type. - tp = sym_get_probable_type(tos); - if (tp == &PySet_Type) { - ADD_OP(_GUARD_TOS_SET, 0, 0); - } - else if (tp == &PyFrozenSet_Type) { - ADD_OP(_GUARD_TOS_FROZENSET, 0, 0); - } + + case _DEOPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - } - op(_GUARD_TOS_SET, (tos -- tos)) { - if (sym_matches_type(tos, &PySet_Type)) { - ADD_OP(_NOP, 0, 0); + case _DEOPT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - else { - sym_set_type(tos, &PySet_Type); + + case _DEOPT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - } - op(_GUARD_TOS_FROZENSET, (tos -- tos)) { - if (sym_matches_type(tos, &PyFrozenSet_Type)) { - ADD_OP(_NOP, 0, 0); + case _HANDLE_PENDING_AND_DEOPT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - else { - sym_set_type(tos, &PyFrozenSet_Type); + + case _HANDLE_PENDING_AND_DEOPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - } - op(_GUARD_TOS_SLICE, (tos -- tos)) { - if (sym_matches_type(tos, &PySlice_Type)) { - ADD_OP(_NOP, 0, 0); + case _HANDLE_PENDING_AND_DEOPT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - else { - sym_set_type(tos, &PySlice_Type); + + case _HANDLE_PENDING_AND_DEOPT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } - } - op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { - if (sym_is_null(null)) { - ADD_OP(_NOP, 0, 0); + case _ERROR_POP_N_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + oparg = CURRENT_OPARG(); + uint32_t target = (uint32_t)CURRENT_OPERAND0_32(); + assert(oparg == 0); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(NULL); } - else { - sym_set_null(null); + + case _SPILL_OR_RELOAD_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { - if (sym_is_not_null(nos)) { - ADD_OP(_NOP, 0, 0); + case _SPILL_OR_RELOAD_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_non_null(nos); + + case _SPILL_OR_RELOAD_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = stack_pointer[-3]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { - if (sym_is_null(null)) { - ADD_OP(_NOP, 0, 0); + case _SPILL_OR_RELOAD_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_null(null); + + case _SPILL_OR_RELOAD_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { - ADD_OP(_NOP, 0, 0); + case _SPILL_OR_RELOAD_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_const(callable, (PyObject *)&PyType_Type); + + case _SPILL_OR_RELOAD_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { - ADD_OP(_NOP, 0, 0); + case _SPILL_OR_RELOAD_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_const(callable, (PyObject *)&PyTuple_Type); + + case _SPILL_OR_RELOAD_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { - ADD_OP(_NOP, 0, 0); + case _SPILL_OR_RELOAD_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - sym_set_const(callable, (PyObject *)&PyUnicode_Type); + + case _SPILL_OR_RELOAD_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache0 = _stack_item_2; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_CALL_LEN, (callable, null, arg -- res, a, c)) { - res = sym_new_type(ctx, &PyLong_Type); - Py_ssize_t length = sym_tuple_length(arg); + case _SPILL_OR_RELOAD_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache1 = _stack_item_2; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - // Not a tuple, check if it's a const string - if (length < 0 && sym_is_const(ctx, arg)) { - PyObject *const_val = sym_get_const(ctx, arg); - if (const_val != NULL) { - if (PyUnicode_CheckExact(const_val)) { - length = PyUnicode_GET_LENGTH(const_val); - } - else if (PyBytes_CheckExact(const_val)) { - length = PyBytes_GET_SIZE(const_val); - } + case _TIER2_RESUME_CHECK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker != iversion) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - if (length >= 0) { - PyObject *temp = PyLong_FromSsize_t(length); - if (temp == NULL) { - goto error; + case _TIER2_RESUME_CHECK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - if (_Py_IsImmortal(temp)) { - ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker != iversion) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - a = arg; - c = callable; - } - op(_GET_LEN, (obj -- obj, len)) { - Py_ssize_t tuple_length = sym_tuple_length(obj); - if (tuple_length == -1) { - len = sym_new_type(ctx, &PyLong_Type); - } - else { - assert(tuple_length >= 0); - PyObject *temp = PyLong_FromSsize_t(tuple_length); - if (temp == NULL) { - goto error; + case _TIER2_RESUME_CHECK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - if (_Py_IsImmortal(temp)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker != iversion) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - len = sym_new_const(ctx, temp); - Py_DECREF(temp); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { - PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (sym_get_const(ctx, callable) == len) { - ADD_OP(_NOP, 0, 0); - } - else { - sym_set_const(callable, len); + case _TIER2_RESUME_CHECK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker != iversion) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { - PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (sym_get_const(ctx, callable) == isinstance) { - ADD_OP(_NOP, 0, 0); - } - else { - sym_set_const(callable, isinstance); + case _COLD_EXIT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_BackoffCounter temperature = exit->temperature; + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + SET_CURRENT_CACHED_VALUES(0); + TIER2_TO_TIER2(exit->executor); + } + else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, stack_pointer, chain_depth, exit, target->op.arg, previous_executor); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(target); + } } - } - op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { - PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (sym_get_const(ctx, callable) == list_append) { - ADD_OP(_NOP, 0, 0); - } - else { - sym_set_const(callable, list_append); + case _COLD_DYNAMIC_EXIT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _Py_CODEUNIT *target = frame->instr_ptr; + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(target); } - } - op(_BINARY_SLICE, (container, start, stop -- res)) { - // Slicing a string/list/tuple always returns the same type. - PyTypeObject *type = sym_get_type(container); - if (type == &PyUnicode_Type || - type == &PyList_Type || - type == &PyTuple_Type) - { - res = sym_new_type(ctx, type); - } - else { - res = sym_new_not_null(ctx); + case _GUARD_CODE_VERSION__PUSH_FRAME_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_GLOBALS_VERSION, (version/1 --)) { - if (ctx->frame->func != NULL) { - PyObject *globals = ctx->frame->func->func_globals; - if (incorrect_keys(globals, version)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - ctx->done = true; - } - else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - /* Do nothing */ - } - else { - if (!ctx->frame->globals_watched) { - PyDict_Watch(GLOBALS_WATCHER_ID, globals); - _Py_BloomFilter_Add(dependencies, globals); - ctx->frame->globals_watched = true; - } - if (ctx->frame->globals_checked_version == version) { - ADD_OP(_NOP, 0, 0); + case _GUARD_CODE_VERSION__PUSH_FRAME_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - ctx->frame->globals_checked_version = version; - } - op(_LOAD_GLOBAL_BUILTINS, (version/1, index/1 -- res)) { - (void)version; - (void)index; - PyObject *cnst = NULL; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *builtins = interp->builtins; - if (incorrect_keys(builtins, version)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - ctx->done = true; - } - else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { - /* Do nothing */ + case _GUARD_CODE_VERSION__PUSH_FRAME_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - if (!ctx->builtins_watched) { - PyDict_Watch(BUILTINS_WATCHER_ID, builtins); - ctx->builtins_watched = true; + + case _GUARD_CODE_VERSION__PUSH_FRAME_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } } - if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { - cnst = convert_global_to_const(this_instr, builtins); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CODE_VERSION_YIELD_VALUE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - if (cnst == NULL) { - res = sym_new_not_null(ctx); + + case _GUARD_CODE_VERSION_YIELD_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - if (_Py_IsImmortal(cnst)) { - res = PyJitRef_Borrow(sym_new_const(ctx, cnst)); + + case _GUARD_CODE_VERSION_YIELD_VALUE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } } - else { - res = sym_new_const(ctx, cnst); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CODE_VERSION_YIELD_VALUE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res)) { - (void)index; - PyObject *cnst = NULL; - if (ctx->frame->func != NULL) { - PyObject *globals = ctx->frame->func->func_globals; - if (incorrect_keys(globals, version)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - ctx->done = true; + case _GUARD_CODE_VERSION_RETURN_VALUE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } } - else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - /* Do nothing */ + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CODE_VERSION_RETURN_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } } - else { - if (!ctx->frame->globals_watched) { - PyDict_Watch(GLOBALS_WATCHER_ID, globals); - _Py_BloomFilter_Add(dependencies, globals); - ctx->frame->globals_watched = true; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CODE_VERSION_RETURN_VALUE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(uop_buffer_last(&ctx->out_buffer), _GUARD_GLOBALS_VERSION, 0, version); - ctx->frame->globals_checked_version = version; + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CODE_VERSION_RETURN_VALUE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - if (ctx->frame->globals_checked_version == version) { - cnst = convert_global_to_const(this_instr, globals); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_CODE_VERSION_RETURN_GENERATOR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - if (cnst == NULL) { - res = sym_new_not_null(ctx); + + case _GUARD_CODE_VERSION_RETURN_GENERATOR_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - if (_Py_IsImmortal(cnst)) { - res = PyJitRef_Borrow(sym_new_const(ctx, cnst)); + + case _GUARD_CODE_VERSION_RETURN_GENERATOR_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } } - else { - res = sym_new_const(ctx, cnst); - } - } - } - - op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res, ls, ss)) { - res = sym_new_unknown(ctx); - ls = list_st; - ss = sub_st; - } - - op(_MAKE_FUNCTION, (codeobj_st -- func, co)) { - func = sym_new_type(ctx, &PyFunction_Type); - co = codeobj_st; - } - - op(_MATCH_CLASS, (subject, type, names -- attrs, s, tp, n)) { - attrs = sym_new_not_null(ctx); - s = subject; - tp = type; - n = names; - } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1], upd)) { - (void)dict; - upd = update; - } - - op(_RECORD_TOS, (tos -- tos)) { - sym_set_recorded_value(tos, (PyObject *)this_instr->operand0); - } + case _GUARD_CODE_VERSION_RETURN_GENERATOR_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + assert(PyCode_Check(code)); + if (((PyCodeObject *)code)->co_version != version) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_RECORD_TOS_TYPE, (tos -- tos)) { - PyTypeObject *tp = (PyTypeObject *)this_instr->operand0; - sym_set_recorded_type(tos, tp); - } + case _GUARD_IP__PUSH_FRAME_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr; + if (target != (_Py_CODEUNIT *)ip) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_RECORD_NOS, (nos, tos -- nos, tos)) { - sym_set_recorded_value(nos, (PyObject *)this_instr->operand0); - } + case _GUARD_IP__PUSH_FRAME_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr; + if (target != (_Py_CODEUNIT *)ip) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_RECORD_4OS, (value, _3os, nos, tos -- value, _3os, nos, tos)) { - sym_set_recorded_value(value, (PyObject *)this_instr->operand0); - } + case _GUARD_IP__PUSH_FRAME_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr; + if (target != (_Py_CODEUNIT *)ip) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { - sym_set_recorded_value(func, (PyObject *)this_instr->operand0); - } + case _GUARD_IP__PUSH_FRAME_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr; + if (target != (_Py_CODEUNIT *)ip) { + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_RECORD_NOS_GEN_FUNC, (nos, tos -- nos, tos)) { - PyFunctionObject *func = (PyFunctionObject *)this_instr->operand0; - assert(func == NULL || PyFunction_Check(func)); - sym_set_recorded_gen_func(nos, func); - } + case _GUARD_IP_YIELD_VALUE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_RECORD_3OS_GEN_FUNC, (gen, nos, tos -- gen, nos, tos)) { - PyFunctionObject *func = (PyFunctionObject *)this_instr->operand0; - assert(func == NULL || PyFunction_Check(func)); - sym_set_recorded_gen_func(gen, func); - } + case _GUARD_IP_YIELD_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } - op(_GUARD_CODE_VERSION__PUSH_FRAME, (version/2 -- )) { - PyCodeObject *co = get_current_code_object(ctx); - if (co->co_version == version) { - _Py_BloomFilter_Add(dependencies, co); - // Functions derive their version from code objects. - if (sym_get_func_version(ctx->frame->callable) == version) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_YIELD_VALUE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - else { - ctx->done = true; + + case _GUARD_IP_YIELD_VALUE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CODE_VERSION_RETURN_VALUE, (version/2 -- )) { - (void)version; - if (ctx->frame->caller) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_RETURN_VALUE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CODE_VERSION_YIELD_VALUE, (version/2 -- )) { - (void)version; - if (ctx->frame->caller) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_RETURN_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_CODE_VERSION_RETURN_GENERATOR, (version/2 -- )) { - (void)version; - if (ctx->frame->caller) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_RETURN_VALUE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { - (void)ip; - stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); - if (sym_get_func_version(ctx->frame->callable) != 0 && - // We can remove this guard for simple function call targets. - (((PyCodeObject *)ctx->frame->func->func_code)->co_flags & - (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == 0) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_RETURN_VALUE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - } - op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { - (void)ip; - if (ctx->frame->caller) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_RETURN_GENERATOR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); - } - op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { - (void)ip; - if (ctx->frame->caller) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_RETURN_GENERATOR_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); - } - op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { - (void)ip; - if (ctx->frame->caller) { - REPLACE_OP(this_instr, _NOP, 0, 0); + case _GUARD_IP_RETURN_GENERATOR_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; } - stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); - } + case _GUARD_IP_RETURN_GENERATOR_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); + _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + /* _TRACE_RECORD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ -// END BYTECODES // -} +#undef TIER_TWO diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index af73090ad998a4..8b9bc3737260d9 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1132,16 +1132,12 @@ def add_macro( macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] ) -> None: parts: list[Part] = [] - # Counts only real OpName entries (not CacheEffect/flush) so we - # know the exact position of each concrete uop inside the macro. - # CacheEffect → becomes Skip; flush → becomes Flush. - # Neither increments uop_index because neither is a "real" uop. - uop_index = 0 + # Tracks the last real Uop seen; CacheEffect and flush leave it unchanged. + prev_uop: Uop | None = None for part in macro.uops: match part: case parser.OpName(): if part.name == "flush": - # flush is structural, not a real uop; leave uop_index alone. parts.append(Flush()) else: if part.name not in uops: @@ -1150,29 +1146,20 @@ def add_macro( ) uop = uops[part.name] if uop.properties.records_value: - # A recording uop is legal in exactly two positions: - # 1. It is the very first real uop (uop_index == 0). - # 2. It is at index 1 AND the immediately preceding - # real uop is a specializing uop, identified by - # the "_SPECIALIZE_" name prefix. - # (Specializing uops are Tier-1-only; recording - # uops are Tier-2-only — they are orthogonal at - # runtime, so this ordering is safe.) + # Valid if first real uop, or if the previous real uop + # is specializing (cache slots between them are transparent). preceding_is_specializing = ( - uop_index == 1 - and isinstance(parts[-1], Uop) - and parts[-1].name.startswith("_SPECIALIZE_") + prev_uop is not None + and "specializing" in prev_uop.annotations ) - if uop_index != 0 and not preceding_is_specializing: + if prev_uop is not None and not preceding_is_specializing: raise analysis_error( f"Recording uop {part.name} must be first in macro " f"or immediately follow a specializing uop", macro.tokens[0]) parts.append(uop) - uop_index += 1 + prev_uop = uop # flush and CacheEffect intentionally excluded case parser.CacheEffect(): - # Cache-entry skips are structural; they do not occupy a uop - # slot, so uop_index is not incremented. parts.append(Skip(part.size)) case _: assert False diff --git a/Tools/cases_generator/test_analyzer.py b/Tools/cases_generator/test_analyzer.py new file mode 100644 index 00000000000000..7d76a988f40715 --- /dev/null +++ b/Tools/cases_generator/test_analyzer.py @@ -0,0 +1,141 @@ +"""Tests for analyzer.py — specifically the add_macro() recording-uop placement rules. + +Run with: + cd Tools/cases_generator + python -m pytest test_analyzer.py -v +or: + python test_analyzer.py +""" + +import sys +import os +import unittest + +# The cases_generator directory is not on sys.path when invoked from the repo +# root, so add it explicitly. +sys.path.insert(0, os.path.dirname(__file__)) + +import parsing +from analyzer import analyze_forest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _parse(src: str) -> list[parsing.AstNode]: + """Parse a raw DSL string (no BEGIN/END markers needed) into an AST forest.""" + psr = parsing.Parser(src, filename="") + nodes: list[parsing.AstNode] = [] + while node := psr.definition(): + nodes.append(node) # type: ignore[arg-type] + return nodes + + +def _analyze(src: str): + """Parse *src* and run analyze_forest(); return the Analysis object.""" + return analyze_forest(_parse(src)) + + +# --------------------------------------------------------------------------- +# Shared DSL fragments +# --------------------------------------------------------------------------- + +# A minimal specializing op (tier == 1 because of the "specializing" annotation). +_SPECIALIZE_OP = """\ +specializing op(_SPECIALIZE_DUMMY, (counter/1, value -- value)) { +} +""" + +# A minimal recording op: uses RECORD_VALUE → records_value == True. +_RECORD_OP = """\ +op(_RECORD_DUMMY, (value -- value)) { + RECORD_VALUE(PyStackRef_AsPyObjectBorrow(value)); +} +""" + +# A plain (non-specializing, non-recording) worker op. +_WORKER_OP = """\ +op(_WORKER_DUMMY, (value -- res)) { + res = value; +} +""" + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestAnalyzer(unittest.TestCase): + + def test_recording_uop_position(self) -> None: + """Recording uops must be first, or immediately follow a specializing uop. + + Case 1 — VALID: recording uop directly after specializing uop. + Case 2 — VALID: recording uop after specializing uop with a cache effect + (unused/1) between them; cache effects are transparent. + Case 3 — INVALID: recording uop after a plain (non-specializing) worker uop. + """ + + # ------------------------------------------------------------------ + # Case 1: _SPECIALIZE_DUMMY + _RECORD_DUMMY (no cache between them) + # ------------------------------------------------------------------ + src_valid_direct = ( + _SPECIALIZE_OP + + _RECORD_OP + + _WORKER_OP + + "macro(VALID_DIRECT) = _SPECIALIZE_DUMMY + _RECORD_DUMMY + _WORKER_DUMMY;\n" + ) + # Must not raise — the recording uop follows the specializing uop directly. + try: + _analyze(src_valid_direct) + except SyntaxError as exc: + self.fail( + f"Case 1 (valid: recording after specializing) raised unexpectedly: {exc}" + ) + + # ------------------------------------------------------------------ + # Case 2: _SPECIALIZE_DUMMY + unused/1 + _RECORD_DUMMY + # A CacheEffect between them must be transparent. + # ------------------------------------------------------------------ + src_valid_with_cache = ( + _SPECIALIZE_OP + + _RECORD_OP + + _WORKER_OP + + "macro(VALID_CACHE) = _SPECIALIZE_DUMMY + unused/1 + _RECORD_DUMMY + _WORKER_DUMMY;\n" + ) + try: + _analyze(src_valid_with_cache) + except SyntaxError as exc: + self.fail( + f"Case 2 (valid: recording after specializing + cache) raised unexpectedly: {exc}" + ) + + # ------------------------------------------------------------------ + # Case 3: _WORKER_DUMMY + _RECORD_DUMMY + # A recording uop after a non-specializing uop must be rejected. + # ------------------------------------------------------------------ + src_invalid = ( + _SPECIALIZE_OP + + _RECORD_OP + + _WORKER_OP + + "macro(INVALID) = _WORKER_DUMMY + _RECORD_DUMMY;\n" + ) + with self.assertRaises(SyntaxError) as ctx: + _analyze(src_invalid) + + # Confirm the error message is the one we emit, not some unrelated error. + self.assertIn( + "Recording uop", + str(ctx.exception), + msg="Case 3: SyntaxError message should mention 'Recording uop'", + ) + self.assertIn( + "_RECORD_DUMMY", + str(ctx.exception), + msg="Case 3: SyntaxError message should name the offending uop", + ) + + +if __name__ == "__main__": + unittest.main() From 1bbac01df0106f4085160df21f0314e8be6b362e Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mishra Date: Sat, 11 Apr 2026 21:39:06 +0530 Subject: [PATCH 3/5] Add missing mypy return type annotation in test --- Tools/cases_generator/test_analyzer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/cases_generator/test_analyzer.py b/Tools/cases_generator/test_analyzer.py index 7d76a988f40715..e49ecf416d1cb5 100644 --- a/Tools/cases_generator/test_analyzer.py +++ b/Tools/cases_generator/test_analyzer.py @@ -10,6 +10,7 @@ import sys import os import unittest +from typing import Any # The cases_generator directory is not on sys.path when invoked from the repo # root, so add it explicitly. @@ -32,7 +33,7 @@ def _parse(src: str) -> list[parsing.AstNode]: return nodes -def _analyze(src: str): +def _analyze(src: str) -> Any: """Parse *src* and run analyze_forest(); return the Analysis object.""" return analyze_forest(_parse(src)) From d9df27ecf8b3c1bd301e287f3aee55b83d0584c7 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mishra Date: Sat, 11 Apr 2026 22:50:13 +0530 Subject: [PATCH 4/5] Revert generated C files to upstream state to fix CRLF --- Python/optimizer_bytecodes.c | 24390 +++------------------------------ 1 file changed, 2013 insertions(+), 22377 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 207c95f5dac597..c12a4f4131bc7e 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1,22654 +1,2290 @@ -// This file is generated by Tools/cases_generator/tier2_generator.py -// from: -// Python/bytecodes.c -// Do not edit! - -#ifdef TIER_ONE - #error "This file is for Tier 2 only" -#endif -#define TIER_TWO 2 - - case _NOP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _NOP_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _NOP_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _NOP_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_PERIODIC_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 because it is replaced */ - - case _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _RESUME_CHECK_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #endif - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _RESUME_CHECK_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _RESUME_CHECK_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _RESUME_CHECK_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _LOAD_FAST_CHECK_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - value = PyStackRef_DUP(value_s); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_CHECK_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - value = PyStackRef_DUP(value_s); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_CHECK_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - value = PyStackRef_DUP(value_s); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_0_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_0_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_0_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_1_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_1_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_1_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_2_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_2_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_2_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_3_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_3_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_3_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_4_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_4_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_4_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_5_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_5_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_5_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_6_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_6_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_6_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_7_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_7_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_7_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_0_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_0_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_0_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_1_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_1_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_1_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_2_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_2_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_2_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_3_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_3_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_3_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_4_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_4_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_4_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_5_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_5_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_5_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_6_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_6_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_6_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_7_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_7_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_7_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_AND_CLEAR_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_AND_CLEAR_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FAST_AND_CLEAR_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_CONST_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_CONST_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_CONST_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_0_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_0_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_0_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_1_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_1_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_1_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_2_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_2_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_2_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_3_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_3_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_3_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SMALL_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_0_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_0_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_0_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_0_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_1_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_1_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_1_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_1_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_2_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_2_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_2_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_2_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_3_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_3_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_3_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_3_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_4_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_4_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_4_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_4_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_5_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_5_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_5_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_5_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_6_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_6_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_6_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_6_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_7_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_7_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_7_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_7_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - oparg = CURRENT_OPARG(); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - value = _stack_item_0; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache0 = trash; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - value = _stack_item_1; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache1 = trash; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SWAP_FAST_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef trash; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - value = _stack_item_2; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - trash = tmp; - _tos_cache2 = trash; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_NOP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || - _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_NOP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || - _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_NOP_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || - _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_NOP_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || - _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_INT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_INT_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_INT_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_INT_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_FLOAT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_FLOAT_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_FLOAT_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_FLOAT_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_UNICODE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_UNICODE_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_UNICODE_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_UNICODE_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_TOP_OPARG_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_NULL_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef res; - res = PyStackRef_NULL; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_NULL_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - res = PyStackRef_NULL; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_NULL_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - res = PyStackRef_NULL; - _tos_cache2 = res; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _END_FOR_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_ITER_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef index_or_null; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - index_or_null = _stack_item_1; - iter = _stack_item_0; - (void)index_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _END_SEND_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef index_or_null; - _PyStackRef receiver; - _PyStackRef val; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - index_or_null = _stack_item_1; - receiver = _stack_item_0; - val = value; - (void)index_or_null; - stack_pointer[0] = val; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = val; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NEGATIVE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NEGATIVE_FLOAT_INPLACE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - value = stack_pointer[-1]; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyFloat_CheckExact(val_o)); - assert(_PyObject_IsUniquelyReferenced(val_o)); - STAT_INC(UNARY_NEGATIVE, hit); - double dres = -((PyFloatObject *)val_o)->ob_fval; - ((PyFloatObject *)val_o)->ob_fval = dres; - res = value; - v = PyStackRef_NULL; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NEGATIVE_FLOAT_INPLACE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyFloat_CheckExact(val_o)); - assert(_PyObject_IsUniquelyReferenced(val_o)); - STAT_INC(UNARY_NEGATIVE, hit); - double dres = -((PyFloatObject *)val_o)->ob_fval; - ((PyFloatObject *)val_o)->ob_fval = dres; - res = value; - v = PyStackRef_NULL; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NEGATIVE_FLOAT_INPLACE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyFloat_CheckExact(val_o)); - assert(_PyObject_IsUniquelyReferenced(val_o)); - STAT_INC(UNARY_NEGATIVE, hit); - double dres = -((PyFloatObject *)val_o)->ob_fval; - ((PyFloatObject *)val_o)->ob_fval = dres; - res = value; - v = PyStackRef_NULL; - _tos_cache2 = v; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NOT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NOT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NOT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_NOT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = res; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = err ? PyStackRef_True : PyStackRef_False; - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_BOOL_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_BOOL_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - if (!PyStackRef_BoolCheck(value)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_BOOL_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - if (!PyStackRef_BoolCheck(value)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_BOOL_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - if (!PyStackRef_BoolCheck(value)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_INT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - value = stack_pointer[-1]; - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_INT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; - v = value; - _tos_cache2 = v; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_LIST_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_LIST_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_LIST_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_LIST_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_LIST_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_LIST_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_LIST_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_LIST_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SLICE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SLICE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SLICE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SLICE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_LIST_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_LIST_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_LIST_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - v = value; - _tos_cache2 = v; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_NONE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - if (!PyStackRef_IsNone(value)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_NONE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - if (!PyStackRef_IsNone(value)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_NONE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - if (!PyStackRef_IsNone(value)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_NONE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - if (!PyStackRef_IsNone(value)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - _tos_cache2 = res; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_COMPACT_ASCII_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (!PyUnicode_IS_COMPACT_ASCII(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_COMPACT_ASCII_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (!PyUnicode_IS_COMPACT_ASCII(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_COMPACT_ASCII_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - if (!PyUnicode_IS_COMPACT_ASCII(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_COMPACT_ASCII_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (!PyUnicode_IS_COMPACT_ASCII(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_UNICODE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_UNICODE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_UNICODE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_UNICODE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_UNICODE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_UNICODE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_UNICODE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_UNICODE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_STR_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - value = stack_pointer[-1]; - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_STR_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _TO_BOOL_STR_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; - v = value; - _tos_cache2 = v; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _REPLACE_WITH_TRUE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - value = stack_pointer[-1]; - res = PyStackRef_True; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _REPLACE_WITH_TRUE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - res = PyStackRef_True; - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _REPLACE_WITH_TRUE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - res = PyStackRef_True; - v = value; - _tos_cache2 = v; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNARY_INVERT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - v = value; - _tos_cache1 = v; - _tos_cache0 = res; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_INT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_INT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_INT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_INT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_INT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_INT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_INT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_INT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_OVERFLOWED_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_OVERFLOWED_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_OVERFLOWED_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_OVERFLOWED_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_OVERFLOWED_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_OVERFLOWED_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_OVERFLOWED_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_OVERFLOWED_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_INPLACE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_INPLACE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_INPLACE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_INPLACE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_INPLACE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_INPLACE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_INPLACE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_INPLACE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_INPLACE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply); - if (PyStackRef_IsNull(_int_inplace_res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - res = _int_inplace_res; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_FLOAT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_FLOAT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_FLOAT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_FLOAT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FLOAT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FLOAT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FLOAT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FLOAT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *d = PyFloat_FromDouble(dres); - if (d == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(d); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_INPLACE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - FLOAT_INPLACE_OP(left, right, left, +); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_INPLACE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - FLOAT_INPLACE_OP(left, right, left, +); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_INPLACE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - FLOAT_INPLACE_OP(left, right, left, +); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - FLOAT_INPLACE_OP(left, right, left, -); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - FLOAT_INPLACE_OP(left, right, left, -); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - FLOAT_INPLACE_OP(left, right, left, -); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - FLOAT_INPLACE_OP(left, right, left, *); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - FLOAT_INPLACE_OP(left, right, left, *); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - FLOAT_INPLACE_OP(left, right, left, *); - res = left; - l = PyStackRef_NULL; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - FLOAT_INPLACE_OP(left, right, right, +); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - FLOAT_INPLACE_OP(left, right, right, +); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - FLOAT_INPLACE_OP(left, right, right, +); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - FLOAT_INPLACE_OP(left, right, right, *); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - FLOAT_INPLACE_OP(left, right, right, *); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - FLOAT_INPLACE_OP(left, right, right, *); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - FLOAT_INPLACE_OP(left, right, right, -); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - FLOAT_INPLACE_OP(left, right, right, -); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - FLOAT_INPLACE_OP(left, right, right, -); - res = right; - l = left; - r = PyStackRef_NULL; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_UNICODE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_UNICODE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - if (res_o == NULL) { - stack_pointer[0] = right; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_ADD_UNICODE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - if (res_o == NULL) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_INPLACE_ADD_UNICODE_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = (int)CURRENT_OPERAND0_16(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - assert(PyUnicode_CheckExact(left_o)); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyObject *right_o = PyStackRef_AsPyObjectSteal(right); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyUnicode_Append(&temp, right_o); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_NULL; - if (temp == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(temp); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BINARY_OP_EXTEND_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = right; - _tos_cache0 = left; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_EXTEND_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_SLICE_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef stop; - _PyStackRef start; - _PyStackRef container; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - stop = _stack_item_2; - start = _stack_item_1; - container = _stack_item_0; - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *res_o; - if (PyList_CheckExact(container_o)) { - stack_pointer[0] = container; - stack_pointer[1] = start; - stack_pointer[2] = stop; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = _PyList_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else if (PyTuple_CheckExact(container_o)) { - stack_pointer[0] = container; - stack_pointer[1] = start; - stack_pointer[2] = stop; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else if (PyUnicode_CheckExact(container_o)) { - stack_pointer[0] = container; - stack_pointer[1] = start; - stack_pointer[2] = stop; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - PyObject *slice = PySlice_New(start_o, stop_o, NULL); - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer[0] = container; - stack_pointer[1] = start; - stack_pointer[2] = stop; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(container_o, slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - } - stack_pointer += 3; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = stop; - stop = PyStackRef_NULL; - stack_pointer[-3] = container; - stack_pointer[-2] = start; - stack_pointer[-1] = stop; - PyStackRef_CLOSE(tmp); - tmp = start; - start = PyStackRef_NULL; - stack_pointer[-2] = start; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-3] = container; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_SLICE_r30: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef stop; - _PyStackRef start; - _PyStackRef container; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - stop = _stack_item_2; - start = _stack_item_1; - container = _stack_item_0; - v = stack_pointer[-1]; - stack_pointer[0] = container; - stack_pointer[1] = start; - stack_pointer[2] = stop; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = container; - container = PyStackRef_NULL; - stack_pointer[-3] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-4] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -4; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_LIST_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef res; - _PyStackRef ls; - _PyStackRef ss; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - list_st = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); - if (index < 0) { - index += PyList_GET_SIZE(list); - } - #ifdef Py_GIL_DISABLED - stack_pointer[0] = list_st; - stack_pointer[1] = sub_st; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = list_st; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index < 0 || index >= PyList_GET_SIZE(list)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = list_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += 2; - #endif - STAT_INC(BINARY_OP, hit); - ls = list_st; - ss = sub_st; - _tos_cache2 = ss; - _tos_cache1 = ls; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_LIST_SLICE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef res; - _PyStackRef ls; - _PyStackRef ss; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - list_st = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PySlice_Check(sub)); - assert(PyList_CheckExact(list)); - stack_pointer[0] = list_st; - stack_pointer[1] = sub_st; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - STAT_INC(BINARY_OP, hit); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - ls = list_st; - ss = sub_st; - _tos_cache2 = ss; - _tos_cache1 = ls; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_STR_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef str_st; - _PyStackRef res; - _PyStackRef s; - _PyStackRef i; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - str_st = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - assert(PyLong_CheckExact(sub)); - assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = str_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = str_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; - assert(c < 128); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - s = str_st; - i = sub_st; - res = PyStackRef_FromPyObjectBorrow(res_o); - _tos_cache2 = i; - _tos_cache1 = s; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_USTR_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef str_st; - _PyStackRef res; - _PyStackRef s; - _PyStackRef i; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - str_st = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - assert(PyLong_CheckExact(sub)); - assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = str_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = str_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = str_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - s = str_st; - i = sub_st; - res = PyStackRef_FromPyObjectBorrow(res_o); - _tos_cache2 = i; - _tos_cache1 = s; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TUPLE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TUPLE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TUPLE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TUPLE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_TUPLE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_TUPLE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_TUPLE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_TUPLE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef tuple_st; - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = sub_st; - _tos_cache0 = tuple_st; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef tuple_st; - _PyStackRef _stack_item_0 = _tos_cache0; - sub_st = _stack_item_0; - tuple_st = stack_pointer[-1]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = sub_st; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = sub_st; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = sub_st; - _tos_cache0 = tuple_st; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef tuple_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - tuple_st = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = tuple_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = tuple_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = sub_st; - _tos_cache0 = tuple_st; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef tuple_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - sub_st = _stack_item_2; - tuple_st = _stack_item_1; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = sub_st; - _tos_cache1 = tuple_st; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = sub_st; - _tos_cache1 = tuple_st; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = sub_st; - _tos_cache1 = tuple_st; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_TUPLE_INT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef tuple_st; - _PyStackRef res; - _PyStackRef ts; - _PyStackRef ss; - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - ts = tuple_st; - ss = sub_st; - _tos_cache2 = ss; - _tos_cache1 = ts; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_TUPLE_INT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef tuple_st; - _PyStackRef res; - _PyStackRef ts; - _PyStackRef ss; - _PyStackRef _stack_item_0 = _tos_cache0; - sub_st = _stack_item_0; - tuple_st = stack_pointer[-1]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - ts = tuple_st; - ss = sub_st; - _tos_cache2 = ss; - _tos_cache1 = ts; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_TUPLE_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef tuple_st; - _PyStackRef res; - _PyStackRef ts; - _PyStackRef ss; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - tuple_st = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - ts = tuple_st; - ss = sub_st; - _tos_cache2 = ss; - _tos_cache1 = ts; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_DICT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_DICT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_DICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_DICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_ANY_DICT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_ANY_DICT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_ANY_DICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_ANY_DICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_DICT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_DICT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_DICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_DICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_DICT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_DICT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_DICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_DICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENDICT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENDICT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENDICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENDICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef dict_st; - _PyStackRef res; - _PyStackRef ds; - _PyStackRef ss; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - dict_st = _stack_item_0; - PyObject *hash = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyAnyDict_CheckExact(dict)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o; - stack_pointer[0] = dict_st; - stack_pointer[1] = sub_st; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = _PyDict_GetItemRef_KnownHash((PyDictObject *)dict, sub, (Py_hash_t)hash, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (rc <= 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - ds = dict_st; - ss = sub_st; - _tos_cache2 = ss; - _tos_cache1 = ds; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_DICT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef dict_st; - _PyStackRef res; - _PyStackRef ds; - _PyStackRef ss; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub_st = _stack_item_1; - dict_st = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyAnyDict_CheckExact(dict)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o; - stack_pointer[0] = dict_st; - stack_pointer[1] = sub_st; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (rc <= 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - ds = dict_st; - ss = sub_st; - _tos_cache2 = ss; - _tos_cache1 = ds; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_CHECK_FUNC_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - container = _stack_item_0; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = container; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = container; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = container; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = container; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - getitem = PyStackRef_FromPyObjectNew(getitem_o); - _tos_cache2 = getitem; - _tos_cache1 = _stack_item_1; - _tos_cache0 = container; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_INIT_CALL_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef container; - _PyStackRef new_frame; - getitem = stack_pointer[-1]; - sub = stack_pointer[-2]; - container = stack_pointer[-3]; - STAT_INC(BINARY_OP, hit); - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_INIT_CALL_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef container; - _PyStackRef new_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - getitem = _stack_item_0; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_INIT_CALL_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef container; - _PyStackRef new_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - getitem = _stack_item_1; - sub = _stack_item_0; - container = stack_pointer[-1]; - STAT_INC(BINARY_OP, hit); - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BINARY_OP_SUBSCR_INIT_CALL_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef container; - _PyStackRef new_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - getitem = _stack_item_2; - sub = _stack_item_1; - container = _stack_item_0; - STAT_INC(BINARY_OP, hit); - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LIST_APPEND_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; - _PyStackRef list; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - v = _stack_item_0; - list = stack_pointer[-1 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SET_ADD_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; - _PyStackRef set; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - v = _stack_item_0; - set = stack_pointer[-1 - (oparg-1)]; - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_SUBSCR_r30: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub; - _PyStackRef container; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - sub = _stack_item_2; - container = _stack_item_1; - v = _stack_item_0; - stack_pointer[0] = v; - stack_pointer[1] = container; - stack_pointer[2] = sub; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-3] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_SUBSCR_LIST_INT_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef value; - _PyStackRef ls; - _PyStackRef ss; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - sub_st = _stack_item_2; - list_st = _stack_item_1; - value = _stack_item_0; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); - if (!LOCK_OBJECT(list)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = sub_st; - _tos_cache1 = list_st; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t len = PyList_GET_SIZE(list); - if (index < 0) { - index += len; - } - if (index < 0 || index >= len) { - UNLOCK_OBJECT(list); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = sub_st; - _tos_cache1 = list_st; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - ls = list_st; - ss = sub_st; - stack_pointer[0] = ls; - stack_pointer[1] = ss; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache1 = ss; - _tos_cache0 = ls; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_SUBSCR_DICT_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub; - _PyStackRef dict_st; - _PyStackRef value; - _PyStackRef st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - sub = _stack_item_2; - dict_st = _stack_item_1; - value = _stack_item_0; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(STORE_SUBSCR, hit); - stack_pointer[0] = value; - stack_pointer[1] = dict_st; - stack_pointer[2] = sub; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - st = dict_st; - _tos_cache0 = st; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_SUBSCR_DICT_KNOWN_HASH_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub; - _PyStackRef dict_st; - _PyStackRef value; - _PyStackRef st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - sub = _stack_item_2; - dict_st = _stack_item_1; - value = _stack_item_0; - PyObject *hash = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(STORE_SUBSCR, hit); - stack_pointer[0] = value; - stack_pointer[1] = dict_st; - stack_pointer[2] = sub; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2_KnownHash((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value), - (Py_hash_t)hash); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - st = dict_st; - _tos_cache0 = st; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DELETE_SUBSCR_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef sub; - _PyStackRef container; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - sub = _stack_item_1; - container = _stack_item_0; - stack_pointer[0] = container; - stack_pointer[1] = sub; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_INTRINSIC_1_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - value = _stack_item_0; - assert(oparg <= MAX_INTRINSIC_1); - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - v = value; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache1 = v; - _tos_cache0 = res; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_INTRINSIC_2_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value1_st; - _PyStackRef value2_st; - _PyStackRef res; - _PyStackRef vs1; - _PyStackRef vs2; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - value1_st = _stack_item_1; - value2_st = _stack_item_0; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - stack_pointer[0] = value2_st; - stack_pointer[1] = value1_st; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - vs1 = value1_st; - vs2 = value2_st; - _tos_cache2 = vs2; - _tos_cache1 = vs1; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAKE_HEAP_SAFE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - value = PyStackRef_MakeHeapSafe(value); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAKE_HEAP_SAFE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - value = PyStackRef_MakeHeapSafe(value); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAKE_HEAP_SAFE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - value = PyStackRef_MakeHeapSafe(value); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAKE_HEAP_SAFE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - value = PyStackRef_MakeHeapSafe(value); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _RETURN_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef retval; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - retval = _stack_item_0; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = retval; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GET_AITER_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef obj; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - obj = _stack_item_0; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - stack_pointer[0] = obj; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - stack_pointer[0] = obj; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _tos_cache0 = iter; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GET_ANEXT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef aiter; - _PyStackRef awaitable; - _PyStackRef _stack_item_0 = _tos_cache0; - aiter = _stack_item_0; - stack_pointer[0] = aiter; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - _tos_cache1 = awaitable; - _tos_cache0 = aiter; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GET_AWAITABLE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - iterable = _stack_item_0; - stack_pointer[0] = iterable; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _tos_cache0 = iter; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _SEND_GEN_FRAME_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; - _PyStackRef receiver; - _PyStackRef gen_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - v = _stack_item_2; - receiver = _stack_item_0; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = v; - _tos_cache1 = _stack_item_1; - _tos_cache0 = receiver; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (!gen_try_set_executing((PyGenObject *)gen)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = v; - _tos_cache1 = _stack_item_1; - _tos_cache0 = receiver; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(SEND, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; - gen_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache2 = gen_frame; - _tos_cache1 = _stack_item_1; - _tos_cache0 = receiver; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _YIELD_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef retval; - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - retval = _stack_item_0; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - _PyStackRef temp = retval; - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - _tos_cache0 = value; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_EXCEPT_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef exc_value; - _PyStackRef _stack_item_0 = _tos_cache0; - exc_value = _stack_item_0; - _PyErr_StackItem *exc_info = tstate->exc_info; - stack_pointer[0] = exc_value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_COMMON_CONSTANT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_COMMON_CONSTANT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_COMMON_CONSTANT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_BUILD_CLASS_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bc; - int err; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - _tos_cache0 = bc; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_NAME_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - v = _stack_item_0; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (PyDict_CheckExact(ns)) { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DELETE_NAME_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef *top; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - seq = _stack_item_0; - top = &stack_pointer[oparg]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_TWO_TUPLE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - seq = _stack_item_0; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != 2) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = seq; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[0] = val1; - stack_pointer[1] = val0; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache1 = val0; - _tos_cache0 = val1; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - seq = stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); - PyObject_GC_UnTrack(seq_o); - _PyStolenTuple_Free(seq_o); - _tos_cache1 = val0; - _tos_cache0 = val1; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - _PyStackRef _stack_item_0 = _tos_cache0; - seq = _stack_item_0; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); - PyObject_GC_UnTrack(seq_o); - _PyStolenTuple_Free(seq_o); - _tos_cache1 = val0; - _tos_cache0 = val1; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - seq = _stack_item_1; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); - PyObject_GC_UnTrack(seq_o); - _PyStolenTuple_Free(seq_o); - _tos_cache2 = val0; - _tos_cache1 = val1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef val2; - _PyStackRef val1; - _PyStackRef val0; - seq = stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); - val2 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 2)); - PyObject_GC_UnTrack(seq_o); - _PyStolenTuple_Free(seq_o); - _tos_cache2 = val0; - _tos_cache1 = val1; - _tos_cache0 = val2; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef val2; - _PyStackRef val1; - _PyStackRef val0; - _PyStackRef _stack_item_0 = _tos_cache0; - seq = _stack_item_0; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); - val2 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 2)); - PyObject_GC_UnTrack(seq_o); - _PyStolenTuple_Free(seq_o); - _tos_cache2 = val0; - _tos_cache1 = val1; - _tos_cache0 = val2; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_TUPLE_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef *values; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - seq = _stack_item_0; - values = &stack_pointer[0]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = seq; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - stack_pointer += oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef *values; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - seq = _stack_item_0; - values = &stack_pointer[0]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - assert(PyTuple_CheckExact(seq_o)); - assert(PyTuple_GET_SIZE(seq_o) == oparg); - assert(_PyObject_IsUniquelyReferenced(seq_o)); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectSteal(items[i]); - } - PyObject_GC_UnTrack(seq_o); - _PyStolenTuple_Free(seq_o); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_SEQUENCE_LIST_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef *values; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - seq = _stack_item_0; - values = &stack_pointer[0]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyList_CheckExact(seq_o)); - if (!LOCK_OBJECT(seq_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = seq; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = seq; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); - stack_pointer += oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _UNPACK_EX_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef seq; - _PyStackRef *top; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - seq = _stack_item_0; - top = &stack_pointer[1 + (oparg & 0xFF) + (oparg >> 8)]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_ATTR_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - owner = _stack_item_1; - v = _stack_item_0; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - stack_pointer[0] = v; - stack_pointer[1] = owner; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = owner; - owner = PyStackRef_NULL; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-2] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DELETE_ATTR_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - stack_pointer[0] = owner; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_GLOBAL_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - v = _stack_item_0; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DELETE_GLOBAL_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_LOCALS_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - locals = PyStackRef_FromPyObjectNew(l); - _tos_cache0 = locals; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_LOCALS_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef locals; - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *l = LOCALS(); - if (l == NULL) { - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - locals = PyStackRef_FromPyObjectNew(l); - _tos_cache1 = locals; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_LOCALS_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef locals; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *l = LOCALS(); - if (l == NULL) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - locals = PyStackRef_FromPyObjectNew(l); - _tos_cache2 = locals; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - - case _LOAD_NAME_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (l_v == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(v_o); - Py_DECREF(l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - v = PyStackRef_FromPyObjectSteal(v_o); - _tos_cache0 = v; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_GLOBAL_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *res; - oparg = CURRENT_OPARG(); - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_NULL_CONDITIONAL_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *null; - oparg = CURRENT_OPARG(); - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += (oparg & 1); - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_GLOBALS_VERSION_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_GLOBALS_VERSION_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_GLOBALS_VERSION_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_GLOBALS_VERSION_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_GLOBAL_MODULE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); - uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_GLOBAL_BUILTINS_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); - uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DELETE_FAST_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAKE_CELL_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DELETE_DEREF_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_FROM_DICT_OR_DEREF_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef class_dict_st; - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - class_dict_st = _stack_item_0; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - int err; - stack_pointer[0] = class_dict_st; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - _tos_cache0 = value; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_DEREF_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(value)) { - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = value; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_DEREF_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - v = _stack_item_0; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_FREE_VARS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_FREE_VARS_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_FREE_VARS_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_FREE_VARS_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_STRING_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *pieces; - _PyStackRef str; - oparg = CURRENT_OPARG(); - pieces = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (str_o == NULL) { - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - str = PyStackRef_FromPyObjectSteal(str_o); - _tos_cache0 = str; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_INTERPOLATION_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *format; - _PyStackRef str; - _PyStackRef value; - _PyStackRef interpolation; - oparg = CURRENT_OPARG(); - format = &stack_pointer[-(oparg & 1)]; - str = stack_pointer[-1 - (oparg & 1)]; - value = stack_pointer[-2 - (oparg & 1)]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); - int conversion = oparg >> 2; - PyObject *format_o; - if (oparg & 1) { - format_o = PyStackRef_AsPyObjectBorrow(format[0]); - } - else { - format_o = &_Py_STR(empty); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (oparg & 1) { - stack_pointer += -(oparg & 1); - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer += -(oparg & 1); - } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (interpolation_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); - _tos_cache0 = interpolation; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_TEMPLATE_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef interpolations; - _PyStackRef strings; - _PyStackRef template; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - interpolations = _stack_item_1; - strings = _stack_item_0; - PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); - PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); - stack_pointer[0] = strings; - stack_pointer[1] = interpolations; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (template_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - template = PyStackRef_FromPyObjectSteal(template_o); - _tos_cache0 = template; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_TUPLE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *values; - _PyStackRef tup; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - tup = PyStackRef_FromPyObjectStealMortal(tup_o); - _tos_cache0 = tup; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_LIST_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *values; - _PyStackRef list; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (list_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - list = PyStackRef_FromPyObjectStealMortal(list_o); - _tos_cache0 = list; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LIST_EXTEND_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iterable_st; - _PyStackRef list_st; - _PyStackRef i; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - iterable_st = _stack_item_0; - list_st = stack_pointer[-1 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - stack_pointer[0] = iterable_st; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - assert(Py_IsNone(none_val)); - i = iterable_st; - _tos_cache0 = i; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SET_UPDATE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iterable; - _PyStackRef set; - _PyStackRef i; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - iterable = _stack_item_0; - set = stack_pointer[-1 - (oparg-1)]; - stack_pointer[0] = iterable; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - i = iterable; - _tos_cache0 = i; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_SET_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *values; - _PyStackRef set; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - int err = 0; - for (Py_ssize_t i = 0; i < oparg; i++) { - _PyStackRef value = values[i]; - values[i] = PyStackRef_NULL; - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - if (err) { - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - set = PyStackRef_FromPyObjectStealMortal(set_o); - _tos_cache0 = set; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_MAP_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *values; - _PyStackRef map; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg*2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (map_o == NULL) { - stack_pointer += -oparg*2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - map = PyStackRef_FromPyObjectStealMortal(map_o); - _tos_cache0 = map; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -oparg*2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SETUP_ANNOTATIONS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - int err; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DICT_UPDATE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef update; - _PyStackRef dict; - _PyStackRef upd; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - update = _stack_item_0; - dict = stack_pointer[-1 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - stack_pointer[0] = update; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *exc = _PyErr_GetRaisedException(tstate); - int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (has_keys == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%T' object is not a mapping", - update_o); - Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_ChainExceptions1(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - upd = update; - _tos_cache0 = upd; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _DICT_MERGE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef update; - _PyStackRef dict; - _PyStackRef callable; - _PyStackRef u; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - update = _stack_item_0; - dict = stack_pointer[-1 - (oparg - 1)]; - callable = stack_pointer[-4 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - PyObject *dupkey = NULL; - stack_pointer[0] = update; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); - Py_XDECREF(dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - u = update; - _tos_cache0 = u; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAP_ADD_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef key; - _PyStackRef dict_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - value = _stack_item_1; - key = _stack_item_0; - dict_st = stack_pointer[-1 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - stack_pointer[0] = key; - stack_pointer[1] = value; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SUPER_ATTR_ATTR_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_st; - _PyStackRef class_st; - _PyStackRef global_super_st; - _PyStackRef attr_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - self_st = _stack_item_2; - class_st = _stack_item_1; - global_super_st = _stack_item_0; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = self_st; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (!PyType_Check(class)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = self_st; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer[0] = global_super_st; - stack_pointer[1] = class_st; - stack_pointer[2] = self_st; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (attr == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - attr_st = PyStackRef_FromPyObjectSteal(attr); - _tos_cache0 = attr_st; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TYPE_VERSION_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); - assert(type_version != 0); - if (!PyType_Check((PyObject *)tp)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TYPE_VERSION_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); - assert(type_version != 0); - if (!PyType_Check((PyObject *)tp)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TYPE_VERSION_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); - assert(type_version != 0); - if (!PyType_Check((PyObject *)tp)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_TYPE_VERSION_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(nos); - assert(type_version != 0); - if (!PyType_Check((PyObject *)tp)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_LOAD_SUPER_ATTR_METHOD_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef class_st; - _PyStackRef global_super_st; - oparg = CURRENT_OPARG(); - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (!PyType_Check(class)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_LOAD_SUPER_ATTR_METHOD_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef class_st; - _PyStackRef global_super_st; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - class_st = stack_pointer[-1]; - global_super_st = stack_pointer[-2]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (!PyType_Check(class)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_LOAD_SUPER_ATTR_METHOD_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef class_st; - _PyStackRef global_super_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - class_st = _stack_item_0; - global_super_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = class_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - if (!PyType_Check(class)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = class_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_LOAD_SUPER_ATTR_METHOD_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef class_st; - _PyStackRef global_super_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - class_st = _stack_item_1; - global_super_st = _stack_item_0; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (!PyType_Check(class)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = class_st; - _tos_cache0 = global_super_st; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SUPER_ATTR_METHOD_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_st; - _PyStackRef class_st; - _PyStackRef global_super_st; - _PyStackRef attr; - _PyStackRef self_or_null; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - self_st = _stack_item_2; - class_st = _stack_item_1; - global_super_st = _stack_item_0; - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - PyObject *attr_o; - { - int *method_found_ptr = &method_found; - stack_pointer[0] = global_super_st; - stack_pointer[1] = class_st; - stack_pointer[2] = self_st; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (attr_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = global_super_st; - global_super_st = self_or_null; - stack_pointer[-2] = global_super_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-1] = class_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - attr = PyStackRef_FromPyObjectSteal(attr_o); - _tos_cache1 = self_or_null; - _tos_cache0 = attr; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *self_or_null; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - self_or_null = &stack_pointer[1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - stack_pointer[0] = owner; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(attr)) { - stack_pointer[-1] = attr; - stack_pointer += (oparg&1); - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - } - else { - stack_pointer[0] = owner; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; - stack_pointer += (oparg&1); - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(attr)) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - stack_pointer += -(oparg&1); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[-1] = attr; - stack_pointer += (oparg&1); - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_LOCKED_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_LOCKED_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_LOCKED_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TYPE_VERSION_LOCKED_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_MANAGED_OBJECT_HAS_VALUES_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_MANAGED_OBJECT_HAS_VALUES_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_MANAGED_OBJECT_HAS_VALUES_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_MANAGED_OBJECT_HAS_VALUES_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_INSTANCE_VALUE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - o = owner; - _tos_cache1 = o; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_INSTANCE_VALUE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - o = owner; - _tos_cache1 = o; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_INSTANCE_VALUE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - o = owner; - _tos_cache2 = o; - _tos_cache1 = attr; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_MODULE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint32_t dict_version = (uint32_t)CURRENT_OPERAND0_32(); - uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - o = owner; - _tos_cache1 = o; - _tos_cache0 = attr; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_WITH_HINT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); - assert(PyDict_CheckExact((PyObject *)dict)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #endif - PyObject *attr_o; - if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dk->dk_kind != DICT_KEYS_UNICODE) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; - if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); - if (attr_o == NULL) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - STAT_INC(LOAD_ATTR, hit); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - o = owner; - _tos_cache1 = o; - _tos_cache0 = attr; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_SLOT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - o = owner; - _tos_cache1 = o; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_SLOT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - o = owner; - _tos_cache1 = o; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_SLOT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - o = owner; - _tos_cache2 = o; - _tos_cache1 = attr; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_CLASS_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_CLASS_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_CLASS_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_CLASS_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_CLASS_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[0] = owner; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = attr; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_PROPERTY_FRAME_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef new_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - PyObject *fget = (PyObject *)CURRENT_OPERAND0_64(); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (code->co_kwonlyargcount) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (code->co_argcount != 1) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(LOAD_ATTR, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - pushed_frame->localsplus[0] = owner; - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */ - - case _GUARD_DORV_NO_DICT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_DORV_NO_DICT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_DORV_NO_DICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_DORV_NO_DICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_ATTR_INSTANCE_VALUE_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - value = _stack_item_0; - uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - o = owner; - stack_pointer[0] = o; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = o; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOCK_OBJECT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOCK_OBJECT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOCK_OBJECT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOCK_OBJECT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_ATTR_WITH_HINT_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - owner = _stack_item_1; - value = _stack_item_0; - uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - if (!LOCK_OBJECT(dict)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - UNLOCK_OBJECT(dict); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - stack_pointer[0] = value; - stack_pointer[1] = owner; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - STAT_INC(STORE_ATTR, hit); - o = owner; - stack_pointer[-2] = o; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = o; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _STORE_ATTR_SLOT_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef o; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - value = _stack_item_0; - uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - o = owner; - stack_pointer[0] = o; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = o; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COMPARE_OP_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (oparg & 16) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - } - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COMPARE_OP_FLOAT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - l = left; - r = right; - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COMPARE_OP_FLOAT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - l = left; - r = right; - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COMPARE_OP_FLOAT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - l = left; - r = right; - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COMPARE_OP_INT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - int sign_ish = COMPARISON_BIT(ileft, iright); - l = left; - r = right; - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COMPARE_OP_STR_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - l = left; - r = right; - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _IS_OP_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - _PyStackRef l; - _PyStackRef r; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - b = res ? PyStackRef_True : PyStackRef_False; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = b; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _IS_OP_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - right = _stack_item_0; - left = stack_pointer[-1]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - b = res ? PyStackRef_True : PyStackRef_False; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = b; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _IS_OP_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - b = res ? PyStackRef_True : PyStackRef_False; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = b; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CONTAINS_OP_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = b; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_SET_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_SET_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_SET_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_ANY_SET_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SET_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SET_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SET_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_SET_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENSET_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenSet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENSET_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - tos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenSet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = tos; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENSET_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - tos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenSet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = tos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_FROZENSET_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef tos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - tos = _stack_item_2; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyFrozenSet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = tos; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CONTAINS_OP_SET_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyAnySet_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = b; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CONTAINS_OP_DICT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyAnyDict_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = b; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_EG_MATCH_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef match_type_st; - _PyStackRef exc_value_st; - _PyStackRef rest; - _PyStackRef match; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - match_type_st = _stack_item_1; - exc_value_st = _stack_item_0; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - stack_pointer[0] = exc_value_st; - stack_pointer[1] = match_type_st; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (!Py_IsNone(match_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - _tos_cache1 = match; - _tos_cache0 = rest; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_EXC_MATCH_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); - b = res ? PyStackRef_True : PyStackRef_False; - _tos_cache1 = b; - _tos_cache0 = left; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _IMPORT_NAME_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef fromlist; - _PyStackRef level; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - fromlist = _stack_item_1; - level = _stack_item_0; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyObject *res_o; - if (!(oparg & 0x02)) { - stack_pointer[0] = level; - stack_pointer[1] = fromlist; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), - LOCALS(), name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level), - oparg & 0x01); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer[0] = level; - stack_pointer[1] = fromlist; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), - LOCALS(), name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _IMPORT_FROM_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef from; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - from = _stack_item_0; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o; - if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { - stack_pointer[0] = from; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = _PyEval_LazyImportFrom( - tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer[0] = from; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = _PyEval_ImportFrom( - tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (res_o == NULL) { - stack_pointer[-1] = from; - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache1 = res; - _tos_cache0 = from; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ - - /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ - - case _IS_NONE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef b; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - } - _tos_cache0 = b; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ - - case _GET_LEN_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef obj; - _PyStackRef len; - _PyStackRef _stack_item_0 = _tos_cache0; - obj = _stack_item_0; - stack_pointer[0] = obj; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - len = PyStackRef_FromPyObjectSteal(len_o); - _tos_cache1 = len; - _tos_cache0 = obj; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_CLASS_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef names; - _PyStackRef type; - _PyStackRef subject; - _PyStackRef attrs; - _PyStackRef s; - _PyStackRef tp; - _PyStackRef n; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - names = _stack_item_2; - type = _stack_item_1; - subject = _stack_item_0; - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - stack_pointer[0] = subject; - stack_pointer[1] = type; - stack_pointer[2] = names; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - attrs = PyStackRef_None; - } - s = subject; - tp = type; - n = names; - _tos_cache2 = n; - _tos_cache1 = tp; - _tos_cache0 = s; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer[-3] = attrs; - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_MAPPING_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - _tos_cache1 = res; - _tos_cache0 = subject; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_MAPPING_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef subject; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - subject = _stack_item_0; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - _tos_cache1 = res; - _tos_cache0 = subject; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_MAPPING_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef subject; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - subject = _stack_item_1; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = res; - _tos_cache1 = subject; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_SEQUENCE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - _tos_cache1 = res; - _tos_cache0 = subject; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_SEQUENCE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef subject; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - subject = _stack_item_0; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - _tos_cache1 = res; - _tos_cache0 = subject; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_SEQUENCE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef subject; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - subject = _stack_item_1; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - _tos_cache2 = res; - _tos_cache1 = subject; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MATCH_KEYS_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef keys; - _PyStackRef subject; - _PyStackRef values_or_none; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - keys = _stack_item_1; - subject = _stack_item_0; - stack_pointer[0] = subject; - stack_pointer[1] = keys; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - _tos_cache2 = values_or_none; - _tos_cache1 = keys; - _tos_cache0 = subject; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GET_ITER_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef index_or_null; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - iterable = _stack_item_0; - #ifdef Py_STATS - _Py_GatherStats_GetIter(iterable); - #endif - stack_pointer[0] = iterable; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsError(result)) { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - iter = result; - _tos_cache1 = index_or_null; - _tos_cache0 = iter; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ - - case _FOR_ITER_TIER_TWO_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef next; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null_or_index = _stack_item_1; - iter = _stack_item_0; - stack_pointer[0] = iter; - stack_pointer[1] = null_or_index; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - stack_pointer[-1] = null_or_index; - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - } - next = item; - _tos_cache2 = next; - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ - - case _ITER_CHECK_LIST_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_LIST_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - null_or_index = _stack_item_0; - iter = stack_pointer[-1]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = null_or_index; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = null_or_index; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_LIST_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null_or_index = _stack_item_1; - iter = _stack_item_0; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_LIST_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - null_or_index = _stack_item_2; - iter = _stack_item_1; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ - - case _GUARD_NOT_EXHAUSTED_LIST_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - #ifndef Py_GIL_DISABLED - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_LIST_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - null_or_index = _stack_item_0; - iter = stack_pointer[-1]; - #ifndef Py_GIL_DISABLED - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = null_or_index; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_LIST_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null_or_index = _stack_item_1; - iter = _stack_item_0; - #ifndef Py_GIL_DISABLED - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_LIST_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - null_or_index = _stack_item_2; - iter = _stack_item_1; - #ifndef Py_GIL_DISABLED - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ - - case _ITER_NEXT_LIST_TIER_TWO_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef next; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null_or_index = _stack_item_1; - iter = _stack_item_0; - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(PyList_CheckExact(list_o)); - #ifdef Py_GIL_DISABLED - assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || - _PyObject_GC_IS_SHARED(list_o)); - STAT_INC(FOR_ITER, hit); - stack_pointer[0] = iter; - stack_pointer[1] = null_or_index; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result <= 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - #else - assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); - stack_pointer += 2; - #endif - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - _tos_cache2 = next; - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_TUPLE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_TUPLE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - null_or_index = _stack_item_0; - iter = stack_pointer[-1]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = null_or_index; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_TUPLE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null_or_index = _stack_item_1; - iter = _stack_item_0; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_TUPLE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - null_or_index = _stack_item_2; - iter = _stack_item_1; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ - - case _GUARD_NOT_EXHAUSTED_TUPLE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_TUPLE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - null_or_index = _stack_item_0; - iter = stack_pointer[-1]; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = null_or_index; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_TUPLE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null_or_index = _stack_item_1; - iter = _stack_item_0; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_TUPLE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - null_or_index = _stack_item_2; - iter = _stack_item_1; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = null_or_index; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_NEXT_TUPLE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef next; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - _tos_cache2 = next; - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_NEXT_TUPLE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef next; - _PyStackRef _stack_item_0 = _tos_cache0; - null_or_index = _stack_item_0; - iter = stack_pointer[-1]; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - _tos_cache2 = next; - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_NEXT_TUPLE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef next; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null_or_index = _stack_item_1; - iter = _stack_item_0; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - _tos_cache2 = next; - _tos_cache1 = null_or_index; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_RANGE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - iter = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_RANGE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = _stack_item_0; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_RANGE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - iter = _stack_item_0; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_CHECK_RANGE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - iter = _stack_item_1; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - #endif - _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ - - case _GUARD_NOT_EXHAUSTED_RANGE_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - iter = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_RANGE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_RANGE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - iter = _stack_item_0; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOT_EXHAUSTED_RANGE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - iter = _stack_item_1; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_NEXT_RANGE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef next; - iter = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - next = PyStackRef_FromPyObjectSteal(res); - _tos_cache2 = next; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_NEXT_RANGE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef next; - _PyStackRef _stack_item_0 = _tos_cache0; - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - next = PyStackRef_FromPyObjectSteal(res); - _tos_cache2 = next; - _tos_cache1 = _stack_item_0; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _ITER_NEXT_RANGE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef next; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - iter = _stack_item_0; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - stack_pointer[0] = iter; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - next = PyStackRef_FromPyObjectSteal(res); - _tos_cache2 = next; - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _FOR_ITER_GEN_FRAME_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef gen_frame; - oparg = CURRENT_OPARG(); - iter = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (!gen_try_set_executing((PyGenObject *)gen)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); - gen_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache2 = gen_frame; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _FOR_ITER_GEN_FRAME_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef gen_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - iter = stack_pointer[-1]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (!gen_try_set_executing((PyGenObject *)gen)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); - gen_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache2 = gen_frame; - _tos_cache1 = _stack_item_0; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _FOR_ITER_GEN_FRAME_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; - _PyStackRef gen_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - iter = _stack_item_0; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - if (!gen_try_set_executing((PyGenObject *)gen)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); - gen_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache2 = gen_frame; - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INSERT_NULL_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self; - _PyStackRef *method_and_self; - _PyStackRef _stack_item_0 = _tos_cache0; - self = _stack_item_0; - method_and_self = &stack_pointer[0]; - method_and_self[1] = self; - method_and_self[0] = PyStackRef_NULL; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_SPECIAL_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *method_and_self; - oparg = CURRENT_OPARG(); - method_and_self = &stack_pointer[-2]; - PyObject *name = _Py_SpecialMethods[oparg].name; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err <= 0) { - if (err == 0) { - PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); - const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) - ? _Py_SpecialMethods[oparg].error_suggestion - : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _WITH_EXCEPT_START_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef val; - _PyStackRef lasti; - _PyStackRef exit_self; - _PyStackRef exit_func; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - val = _stack_item_2; - lasti = _stack_item_0; - exit_self = stack_pointer[-1]; - exit_func = stack_pointer[-2]; - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - PyObject *original_tb = tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - assert(PyStackRef_IsTaggedInt(lasti)); - (void)lasti; - PyObject* res_o; - { - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - stack_pointer[0] = lasti; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = val; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = res; - _tos_cache1 = val; - _tos_cache0 = _stack_item_1; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_EXC_INFO_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - _tos_cache1 = new_exc; - _tos_cache0 = prev_exc; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_EXC_INFO_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - _PyStackRef _stack_item_0 = _tos_cache0; - exc = _stack_item_0; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - _tos_cache1 = new_exc; - _tos_cache0 = prev_exc; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_EXC_INFO_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - exc = _stack_item_1; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - _tos_cache2 = new_exc; - _tos_cache1 = prev_exc; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_WITH_VALUES_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache1 = self; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_WITH_VALUES_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache1 = self; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_WITH_VALUES_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache2 = self; - _tos_cache1 = attr; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_NO_DICT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache1 = self; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_NO_DICT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache1 = self; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_NO_DICT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache2 = self; - _tos_cache1 = attr; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - _tos_cache0 = attr; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - _tos_cache0 = attr; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_METHOD_LAZY_DICT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_METHOD_LAZY_DICT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_METHOD_LAZY_DICT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_ATTR_METHOD_LAZY_DICT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_LAZY_DICT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache1 = self; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_LAZY_DICT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache1 = self; - _tos_cache0 = attr; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_ATTR_METHOD_LAZY_DICT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - _tos_cache2 = self; - _tos_cache1 = attr; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAYBE_EXPAND_METHOD_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _PY_FRAME_GENERAL_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (temp == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - new_frame = PyStackRef_Wrap(temp); - _tos_cache0 = new_frame; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_FUNCTION_VERSION_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_FUNCTION_VERSION_INLINE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_FUNCTION_VERSION_INLINE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_FUNCTION_VERSION_INLINE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_FUNCTION_VERSION_INLINE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_METHOD_VERSION_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _EXPAND_METHOD_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(self_or_null)); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_NOT_PY_CALLABLE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_NON_PY_GENERAL_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_VectorCall_StackRefSteal( - callable, - arguments, - total_args, - PyStackRef_NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(PyStackRef_IsNull(self_or_null)); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - STAT_INC(CALL, hit); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_PEP_523_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - if (IS_PEP523_HOOKED(tstate)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_PEP_523_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - if (IS_PEP523_HOOKED(tstate)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_PEP_523_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - if (IS_PEP523_HOOKED(tstate)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_PEP_523_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - if (IS_PEP523_HOOKED(tstate)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_FUNCTION_EXACT_ARGS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_STACK_SPACE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_REMAINING_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - if (tstate->py_recursion_remaining <= 1) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_REMAINING_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - if (tstate->py_recursion_remaining <= 1) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_REMAINING_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - if (tstate->py_recursion_remaining <= 1) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_REMAINING_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - if (tstate->py_recursion_remaining <= 1) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INIT_CALL_PY_EXACT_ARGS_0_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INIT_CALL_PY_EXACT_ARGS_1_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INIT_CALL_PY_EXACT_ARGS_2_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - oparg = 2; - assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INIT_CALL_PY_EXACT_ARGS_3_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - oparg = 3; - assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INIT_CALL_PY_EXACT_ARGS_4_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INIT_CALL_PY_EXACT_ARGS_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - _tos_cache0 = new_frame; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PUSH_FRAME_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef new_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - new_frame = _stack_item_0; - assert(!IS_PEP523_HOOKED(tstate)); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NULL_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NULL_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef _stack_item_0 = _tos_cache0; - null = stack_pointer[-1]; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NULL_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null = _stack_item_0; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NULL_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - null = _stack_item_1; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = null; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = null; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_THIRD_NULL_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - null = stack_pointer[-3]; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_THIRD_NULL_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef _stack_item_0 = _tos_cache0; - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_THIRD_NULL_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - null = stack_pointer[-1]; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_THIRD_NULL_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - null = _stack_item_0; - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = null; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TYPE_1_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TYPE_1_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - callable = stack_pointer[-2]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TYPE_1_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - callable = stack_pointer[-1]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TYPE_1_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - callable = _stack_item_0; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_TYPE_1_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - a = arg; - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_TYPE_1_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - arg = _stack_item_0; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - a = arg; - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_TYPE_1_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - arg = _stack_item_1; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - a = arg; - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_TYPE_1_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - arg = _stack_item_2; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - a = arg; - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_STR_1_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_STR_1_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - callable = stack_pointer[-2]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_STR_1_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - callable = stack_pointer[-1]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_STR_1_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - callable = _stack_item_0; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_STR_1_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - arg = _stack_item_2; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = arg; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - a = arg; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache1 = a; - _tos_cache0 = res; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TUPLE_1_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TUPLE_1_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - callable = stack_pointer[-2]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TUPLE_1_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - callable = stack_pointer[-1]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_TUPLE_1_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - callable = _stack_item_0; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_TUPLE_1_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - arg = _stack_item_2; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = arg; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - a = arg; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache1 = a; - _tos_cache0 = res; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_AND_ALLOCATE_OBJECT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyStackRef_IsNull(self_or_null)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (!PyType_Check(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - self_or_null = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CREATE_INIT_FRAME_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self; - _PyStackRef init; - _PyStackRef init_frame; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - init = stack_pointer[-2 - oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (temp == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - tstate->py_recursion_remaining--; - init_frame = PyStackRef_Wrap(temp); - _tos_cache0 = init_frame; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _EXIT_INIT_CHECK_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef should_be_none; - _PyStackRef _stack_item_0 = _tos_cache0; - should_be_none = _stack_item_0; - if (!PyStackRef_IsNone(should_be_none)) { - stack_pointer[0] = should_be_none; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_BUILTIN_CLASS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyType_Check(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (tp->tp_vectorcall == NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_BUILTIN_CLASS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_CallBuiltinClass_StackRef( - callable, - arguments, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_BUILTIN_O_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyCFunction_CheckExact(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - total_args++; - } - if (total_args != 1) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_BUILTIN_O_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef c; - _PyStackRef s; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyStackRef_IsNull(self_or_null)) { - args--; - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = args[0]; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = s; - _tos_cache1 = c; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_BUILTIN_FAST_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyCFunction_CheckExact(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_BUILTIN_FAST_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_BuiltinCallFast_StackRef( - callable, - arguments, - total_args - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyCFunction_CheckExact(callable_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LEN_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LEN_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - callable = stack_pointer[-2]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LEN_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - callable = stack_pointer[-1]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LEN_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - callable = _stack_item_0; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_LEN_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - arg = _stack_item_2; - callable = _stack_item_0; - STAT_INC(CALL, hit); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - stack_pointer[0] = callable; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = arg; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - a = arg; - c = callable; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_ISINSTANCE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - callable = stack_pointer[-4]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = stack_pointer[-3]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_ISINSTANCE_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = stack_pointer[-2]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_ISINSTANCE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - callable = stack_pointer[-2]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = stack_pointer[-1]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_ISINSTANCE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - callable = stack_pointer[-1]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_ISINSTANCE_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef cls; - _PyStackRef instance; - _PyStackRef null; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - cls = _stack_item_2; - instance = _stack_item_1; - null = _stack_item_0; - callable = stack_pointer[-1]; - STAT_INC(CALL, hit); - PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); - PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); - stack_pointer[0] = null; - stack_pointer[1] = instance; - stack_pointer[2] = cls; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - (void)null; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LIST_APPEND_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LIST_APPEND_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - callable = stack_pointer[-2]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LIST_APPEND_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - callable = stack_pointer[-1]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_LIST_APPEND_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - callable = _stack_item_0; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_LIST_APPEND_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef self; - _PyStackRef callable; - _PyStackRef none; - _PyStackRef c; - _PyStackRef s; - oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!LOCK_OBJECT(self_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = self; - none = PyStackRef_None; - _tos_cache2 = s; - _tos_cache1 = c; - _tos_cache0 = none; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_LIST_APPEND_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef self; - _PyStackRef callable; - _PyStackRef none; - _PyStackRef c; - _PyStackRef s; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - arg = _stack_item_0; - self = stack_pointer[-1]; - callable = stack_pointer[-2]; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!LOCK_OBJECT(self_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = arg; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - if (err) { - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = self; - none = PyStackRef_None; - _tos_cache2 = s; - _tos_cache1 = c; - _tos_cache0 = none; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_LIST_APPEND_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef self; - _PyStackRef callable; - _PyStackRef none; - _PyStackRef c; - _PyStackRef s; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - arg = _stack_item_1; - self = _stack_item_0; - callable = stack_pointer[-1]; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!LOCK_OBJECT(self_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = arg; - _tos_cache0 = self; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - if (err) { - stack_pointer[0] = self; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = self; - none = PyStackRef_None; - _tos_cache2 = s; - _tos_cache1 = c; - _tos_cache0 = none; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_LIST_APPEND_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef self; - _PyStackRef callable; - _PyStackRef none; - _PyStackRef c; - _PyStackRef s; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - arg = _stack_item_2; - self = _stack_item_1; - callable = _stack_item_0; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!LOCK_OBJECT(self_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = arg; - _tos_cache1 = self; - _tos_cache0 = callable; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - if (err) { - stack_pointer[0] = callable; - stack_pointer[1] = self; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = self; - none = PyStackRef_None; - _tos_cache2 = s; - _tos_cache1 = c; - _tos_cache0 = none; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (method->d_method->ml_flags != METH_O) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - total_args++; - } - if (total_args != 2) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyObject *self = PyStackRef_AsPyObjectBorrow( - PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_O_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef c; - _PyStackRef s; - _PyStackRef a; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - } - STAT_INC(CALL, hit); - PyCFunction cfunc = method->d_method->ml_meth; - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = arguments[0]; - a = arguments[1]; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = a; - _tos_cache1 = s; - _tos_cache0 = c; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_LIMIT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - if (_Py_ReachedRecursionLimit(tstate)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_LIMIT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - if (_Py_ReachedRecursionLimit(tstate)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_LIMIT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - if (_Py_ReachedRecursionLimit(tstate)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_RECURSION_LIMIT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - if (_Py_ReachedRecursionLimit(tstate)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_O_INLINE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef c; - _PyStackRef s; - _PyStackRef a; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - callable = stack_pointer[-1 - oparg]; - PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg == 2); - STAT_INC(CALL, hit); - volatile PyCFunction cfunc_v = (PyCFunction)cfunc; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *arg = PyStackRef_AsPyObjectBorrow(args[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = args[0]; - a = args[1]; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = a; - _tos_cache1 = s; - _tos_cache0 = c; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer[-1 - oparg] = res; - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( - callable, - cfunc, - self, - arguments, - total_args - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_st; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_st = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( - callable, - cfunc_v, - self, - args - 1, - oparg + 1 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (method->d_method->ml_flags != METH_NOARGS) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - total_args++; - } - if (total_args != 1) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyObject *self = PyStackRef_AsPyObjectBorrow( - PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_NOARGS_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef c; - _PyStackRef s; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); - if (!PyStackRef_IsNull(self_or_null)) { - args--; - } - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - STAT_INC(CALL, hit); - PyCFunction cfunc = method->d_method->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = args[0]; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = s; - _tos_cache1 = c; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef c; - _PyStackRef s; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - callable = stack_pointer[-1 - oparg]; - PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); - assert(oparg == 1); - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - STAT_INC(CALL, hit); - volatile PyCFunction cfunc_v = (PyCFunction)cfunc; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - c = callable; - s = args[0]; - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = s; - _tos_cache1 = c; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (method->d_method->ml_flags != METH_FASTCALL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - total_args++; - } - if (total_args == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - PyObject *self = PyStackRef_AsPyObjectBorrow( - PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_FAST_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( - callable, - cfunc, - self, - arguments, - total_args - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef self_st; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_st = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(self != NULL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( - callable, - cfunc_v, - self, - args - 1, - oparg + 1 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _MAYBE_EXPAND_METHOD_KW_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - } - _tos_cache0 = _stack_item_0; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _PY_FRAME_KW_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef kwnames; - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef new_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - kwnames = _stack_item_0; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[0] = kwnames; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (temp == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - new_frame = PyStackRef_Wrap(temp); - _tos_cache0 = new_frame; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_FUNCTION_VERSION_KW_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_METHOD_VERSION_KW_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (!PyStackRef_IsNull(null)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _EXPAND_METHOD_KW_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(PyStackRef_IsNull(self_or_null)); - _PyStackRef callable_s = callable; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = _stack_item_0; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_NOT_PY_CALLABLE_KW_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callable; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_KW_NON_PY_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef kwnames; - _PyStackRef *args; - _PyStackRef self_or_null; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - kwnames = _stack_item_0; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - stack_pointer[0] = kwnames; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_VectorCall_StackRefSteal( - callable, - arguments, - total_args, - kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -3 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAKE_CALLARGS_A_TUPLE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef callargs; - _PyStackRef func; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - callargs = _stack_item_1; - func = stack_pointer[-1]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = callargs; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = callargs; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _CHECK_IS_PY_CALLABLE_EX_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - func_st = stack_pointer[-4]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) != &PyFunction_Type) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = stack_pointer[-3]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_PY_CALLABLE_EX_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - _PyStackRef _stack_item_0 = _tos_cache0; - func_st = stack_pointer[-3]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) != &PyFunction_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = stack_pointer[-2]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_PY_CALLABLE_EX_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - func_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) != &PyFunction_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = stack_pointer[-1]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_PY_CALLABLE_EX_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - func_st = stack_pointer[-1]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) != &PyFunction_Type) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _PY_FRAME_EX_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef kwargs_st; - _PyStackRef callargs_st; - _PyStackRef func_st; - _PyStackRef ex_frame; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - kwargs_st = _stack_item_2; - callargs_st = _stack_item_1; - func_st = stack_pointer[-1]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - assert(Py_TYPE(func) == &PyFunction_Type); - assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (new_frame == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - ex_frame = PyStackRef_Wrap(new_frame); - _tos_cache0 = ex_frame; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_NOT_PY_CALLABLE_EX_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - func_st = stack_pointer[-4]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = stack_pointer[-3]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_NOT_PY_CALLABLE_EX_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - _PyStackRef _stack_item_0 = _tos_cache0; - func_st = stack_pointer[-3]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = stack_pointer[-2]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_NOT_PY_CALLABLE_EX_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - func_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = stack_pointer[-1]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_IS_NOT_PY_CALLABLE_EX_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_st; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - func_st = stack_pointer[-1]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CALL_FUNCTION_EX_NON_PY_GENERAL_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef kwargs_st; - _PyStackRef callargs_st; - _PyStackRef null; - _PyStackRef func_st; - _PyStackRef result; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - kwargs_st = _stack_item_2; - callargs_st = _stack_item_1; - null = _stack_item_0; - func_st = stack_pointer[-1]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - (void)null; - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[0] = null; - stack_pointer[1] = callargs_st; - stack_pointer[2] = kwargs_st; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - result = PyStackRef_FromPyObjectSteal(result_o); - _tos_cache0 = result; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _MAKE_FUNCTION_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef codeobj_st; - _PyStackRef func; - _PyStackRef co; - _PyStackRef _stack_item_0 = _tos_cache0; - codeobj_st = _stack_item_0; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - stack_pointer[0] = codeobj_st; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - co = codeobj_st; - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - _tos_cache1 = co; - _tos_cache0 = func; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SET_FUNCTION_ATTRIBUTE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_in; - _PyStackRef attr_st; - _PyStackRef func_out; - oparg = CURRENT_OPARG(); - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - _tos_cache0 = func_out; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SET_FUNCTION_ATTRIBUTE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_in; - _PyStackRef attr_st; - _PyStackRef func_out; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - func_in = _stack_item_0; - attr_st = stack_pointer[-1]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - _tos_cache0 = func_out; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SET_FUNCTION_ATTRIBUTE_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_in; - _PyStackRef attr_st; - _PyStackRef func_out; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - func_in = _stack_item_1; - attr_st = _stack_item_0; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - _tos_cache0 = func_out; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SET_FUNCTION_ATTRIBUTE_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef func_in; - _PyStackRef attr_st; - _PyStackRef func_out; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - func_in = _stack_item_2; - attr_st = _stack_item_1; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - _tos_cache1 = func_out; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _RETURN_GENERATOR_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - assert(STACK_LEVEL() <= 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _BUILD_SLICE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef *args; - _PyStackRef slice; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); - PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (slice_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - slice = PyStackRef_FromPyObjectStealMortal(slice_o); - _tos_cache0 = slice; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; +#include "Python.h" +#include "pycore_optimizer.h" +#include "pycore_uops.h" +#include "pycore_uop_ids.h" +#include "internal/pycore_moduleobject.h" + +#define op(name, ...) /* NAME is ignored */ + +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +/* Shortened forms for convenience */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_get_type _Py_uop_sym_get_type +#define sym_has_type _Py_uop_sym_has_type +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM) +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_new_from_symbol _Py_uop_frame_new_from_symbol +#define frame_pop _Py_uop_frame_pop +#define sym_new_tuple _Py_uop_sym_new_tuple +#define sym_tuple_getitem _Py_uop_sym_tuple_getitem +#define sym_tuple_length _Py_uop_sym_tuple_length +#define sym_is_immortal _Py_uop_symbol_is_immortal +#define sym_new_compact_int _Py_uop_sym_new_compact_int +#define sym_is_compact_int _Py_uop_sym_is_compact_int +#define sym_new_truthiness _Py_uop_sym_new_truthiness +#define sym_new_predicate _Py_uop_sym_new_predicate +#define sym_apply_predicate_narrowing _Py_uop_sym_apply_predicate_narrowing +#define sym_set_recorded_type(SYM, TYPE) _Py_uop_sym_set_recorded_type(ctx, SYM, TYPE) +#define sym_set_recorded_value(SYM, VAL) _Py_uop_sym_set_recorded_value(ctx, SYM, VAL) +#define sym_set_recorded_gen_func(SYM, VAL) _Py_uop_sym_set_recorded_gen_func(ctx, SYM, VAL) +#define sym_get_probable_func_code _Py_uop_sym_get_probable_func_code +#define sym_get_probable_value _Py_uop_sym_get_probable_value +#define sym_get_probable_type _Py_uop_sym_get_probable_type +#define sym_set_stack_depth(DEPTH, SP) _Py_uop_sym_set_stack_depth(ctx, DEPTH, SP) + +extern int +optimize_to_bool( + _PyUOpInstruction *this_instr, + JitOptContext *ctx, + JitOptSymbol *value, + JitOptSymbol **result_ptr, + uint16_t prefix, uint16_t suffix); + +extern void +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit); + +extern PyCodeObject *get_code(_PyUOpInstruction *op); + +static int +dummy_func(void) { + + PyCodeObject *co; + int oparg; + JitOptSymbol *flag; + JitOptSymbol *left; + JitOptSymbol *right; + JitOptSymbol *value; + JitOptSymbol *res; + JitOptSymbol *iter; + JitOptSymbol *top; + JitOptSymbol *bottom; + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame *new_frame; + JitOptContext *ctx; + _PyUOpInstruction *this_instr; + _PyBloomFilter *dependencies; + int modified; + int curr_space; + int max_space; + _PyUOpInstruction *first_valid_check_stack; + _PyUOpInstruction *corresponding_check_stack; + +// BEGIN BYTECODES // + + op(_MAKE_HEAP_SAFE, (value -- value)) { + // eliminate _MAKE_HEAP_SAFE when we *know* the value is immortal + if (sym_is_immortal(PyJitRef_Unwrap(value))) { + ADD_OP(_NOP, 0, 0); + } + value = PyJitRef_StripBorrowInfo(value); + } + + op(_COPY_FREE_VARS, (--)) { + PyCodeObject *co = get_current_code_object(ctx); + if (co == NULL) { + ctx->done = true; + break; + } + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + ctx->frame->locals[offset + i] = sym_new_not_null(ctx); + } + } + + op(_LOAD_FAST_CHECK, (-- value)) { + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + ctx->done = true; + } + assert(!PyJitRef_IsUnique(value)); + } + + op(_LOAD_FAST, (-- value)) { + value = GETLOCAL(oparg); + assert(!PyJitRef_IsUnique(value)); + } + + op(_LOAD_FAST_BORROW, (-- value)) { + value = PyJitRef_Borrow(GETLOCAL(oparg)); + assert(!PyJitRef_IsUnique(value)); + } + + op(_LOAD_FAST_AND_CLEAR, (-- value)) { + value = GETLOCAL(oparg); + JitOptRef temp = sym_new_null(ctx); + GETLOCAL(oparg) = temp; + assert(!PyJitRef_IsUnique(value)); + } + + op(_GUARD_TYPE_VERSION_LOCKED, (type_version/2, owner -- owner)) { + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + ADD_OP(_NOP, 0, 0); + } else { + PyTypeObject *probable_type = sym_get_probable_type(owner); + if (probable_type->tp_version_tag == type_version && sym_set_type_version(owner, type_version)) { + // Promote the probable type version to a known one. + if ((probable_type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)probable_type); + _Py_BloomFilter_Add(dependencies, probable_type); + } + } + } + } + + op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner -- o)) { + (void)offset; + (void)value; + o = owner; + } + + op(_STORE_ATTR_WITH_HINT, (hint/1, value, owner -- o)) { + (void)hint; + (void)value; + o = owner; + } + + op(_SWAP_FAST, (value -- trash)) { + JitOptRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyJitRef_RemoveUnique(value); + trash = tmp; + } + + op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- ls, ss)) { + (void)value; + ls = list_st; + ss = sub_st; + } + + op(_STORE_ATTR_SLOT, (index/1, value, owner -- o)) { + (void)index; + (void)value; + o = owner; + } + + op(_STORE_SUBSCR_DICT, (value, dict_st, sub -- st)) { + PyObject *sub_o = sym_get_const(ctx, sub); + if (sub_o != NULL) { + optimize_dict_known_hash(ctx, dependencies, this_instr, + sub_o, _STORE_SUBSCR_DICT_KNOWN_HASH); } - - case _CONVERT_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef result; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - value = _stack_item_0; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - result = PyStackRef_FromPyObjectSteal(result_o); - _tos_cache0 = result; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _FORMAT_SIMPLE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - stack_pointer[0] = value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); + (void)value; + st = dict_st; + } + + op(_PUSH_NULL, (-- res)) { + res = PyJitRef_Borrow(sym_new_null(ctx)); + } + + op(_GUARD_TOS_INT, (value -- value)) { + if (sym_is_compact_int(value)) { + ADD_OP(_NOP, 0, 0); + } + else { + if (sym_get_type(value) == &PyLong_Type) { + ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(value); + } + } + + op(_GUARD_NOS_INT, (left, unused -- left, unused)) { + if (sym_is_compact_int(left)) { + ADD_OP(_NOP, 0, 0); + } + else { + if (sym_get_type(left) == &PyLong_Type) { + ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(left); + } + } + + op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { + PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); + if (type) { + if (type == sym_get_const(ctx, owner)) { + ADD_OP(_NOP, 0, 0); } else { - res = value; + sym_set_const(owner, type); + } + } + } + + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { + assert(type_version); + assert(this_instr[-1].opcode == _RECORD_TOS_TYPE); + if (sym_matches_type_version(owner, type_version)) { + ADD_OP(_NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } + } + + op(_GUARD_TOS_FLOAT, (value -- value)) { + if (sym_matches_type(value, &PyFloat_Type)) { + ADD_OP(_NOP, 0, 0); + } + sym_set_type(value, &PyFloat_Type); + } + + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { + if (sym_matches_type(left, &PyFloat_Type)) { + ADD_OP(_NOP, 0, 0); + } + sym_set_type(left, &PyFloat_Type); + } + + op(_BINARY_OP, (lhs, rhs -- res, l, r)) { + l = lhs; + r = rhs; + REPLACE_OPCODE_IF_EVALUATES_PURE(lhs, rhs, res); + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); + if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { + // There's something other than an int or float involved: + res = sym_new_unknown(ctx); + } + else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { + // This one's fun... the *type* of the result depends on the + // *values* being exponentiated. However, exponents with one + // constant part are reasonably common, so it's probably worth + // trying to infer some simple cases: + // - A: 1 ** 1 -> 1 (int ** int -> int) + // - B: 1 ** -1 -> 1.0 (int ** int -> float) + // - C: 1.0 ** 1 -> 1.0 (float ** int -> float) + // - D: 1 ** 1.0 -> 1.0 (int ** float -> float) + // - E: -1 ** 0.5 ~> 1j (int ** float -> complex) + // - F: 1.0 ** 1.0 -> 1.0 (float ** float -> float) + // - G: -1.0 ** 0.5 ~> 1j (float ** float -> complex) + if (rhs_float) { + // Case D, E, F, or G... can't know without the sign of the LHS + // or whether the RHS is whole, which isn't worth the effort: + res = sym_new_unknown(ctx); + } + else if (lhs_float) { + // Case C: + res = sym_new_type(ctx, &PyFloat_Type); + } + else if (!sym_is_const(ctx, rhs)) { + // Case A or B... can't know without the sign of the RHS: + res = sym_new_unknown(ctx); + } + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { + // Case B: + res = sym_new_type(ctx, &PyFloat_Type); } - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _FORMAT_WITH_SPEC_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef fmt_spec; - _PyStackRef value; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - fmt_spec = _stack_item_1; - value = _stack_item_0; - stack_pointer[0] = value; - stack_pointer[1] = fmt_spec; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - _PyStackRef tmp = fmt_spec; - fmt_spec = PyStackRef_NULL; - stack_pointer[-1] = fmt_spec; - PyStackRef_CLOSE(tmp); - tmp = value; - value = PyStackRef_NULL; - stack_pointer[-2] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + else { + // Case A: + res = sym_new_type(ctx, &PyLong_Type); } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_1_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1]; - top = PyStackRef_DUP(bottom); - _tos_cache1 = top; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_1_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - _PyStackRef _stack_item_0 = _tos_cache0; - bottom = _stack_item_0; - top = PyStackRef_DUP(bottom); - _tos_cache1 = top; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_1_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - bottom = _stack_item_1; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = bottom; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _COPY_2_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-2]; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COPY_2_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - _PyStackRef _stack_item_0 = _tos_cache0; - bottom = stack_pointer[-1]; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = _stack_item_0; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { + res = sym_new_type(ctx, &PyFloat_Type); } - - case _COPY_2_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - bottom = _stack_item_0; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = _stack_item_1; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (lhs_int && rhs_int) { + res = sym_new_type(ctx, &PyLong_Type); } - - case _COPY_3_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-3]; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = stack_pointer[-2]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + res = sym_new_type(ctx, &PyFloat_Type); } + } - case _COPY_3_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - _PyStackRef _stack_item_0 = _tos_cache0; - bottom = stack_pointer[-2]; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = _stack_item_0; - _tos_cache0 = stack_pointer[-1]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) { + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE, 0, 0); } - - case _COPY_3_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - bottom = stack_pointer[-1]; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE_RIGHT, 0, 0); } + // Result may be a unique compact int or a cached small int + // at runtime. Mark as unique; inplace ops verify at runtime. + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } - case _COPY_3_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - bottom = _stack_item_0; - top = PyStackRef_DUP(bottom); - _tos_cache2 = top; - _tos_cache1 = _stack_item_2; - _tos_cache0 = _stack_item_1; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer[0] = bottom; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE, 0, 0); } - - case _COPY_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bottom; - _PyStackRef top; - oparg = CURRENT_OPARG(); - bottom = stack_pointer[-1 - (oparg-1)]; - top = PyStackRef_DUP(bottom); - _tos_cache0 = top; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, 0, 0); } + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } - case _BINARY_OP_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef rhs; - _PyStackRef lhs; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - rhs = _stack_item_1; - lhs = _stack_item_0; - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - stack_pointer[0] = lhs; - stack_pointer[1] = rhs; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - l = lhs; - r = rhs; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) { + if (PyJitRef_IsUnique(left)) { + REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE, 0, 0); } - - case _SWAP_2_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache1 = top; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (PyJitRef_IsUnique(right)) { + REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, 0, 0); } + res = PyJitRef_MakeUnique(sym_new_compact_int(ctx)); + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } - case _SWAP_2_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - _PyStackRef _stack_item_0 = _tos_cache0; - top = _stack_item_0; - bottom = stack_pointer[-1]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache1 = top; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_BINARY_OP_ADD_FLOAT, (left, right -- res, l, r)) { + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; } - - case _SWAP_2_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - top = _stack_item_1; - bottom = _stack_item_0; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache1 = top; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); } - - case _SWAP_2_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - top = _stack_item_2; - bottom = _stack_item_1; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache2 = top; - _tos_cache1 = bottom; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + l = left; + r = right; } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); + } - case _SWAP_3_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-3]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache2 = top; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res, l, r)) { + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; } - - case _SWAP_3_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - _PyStackRef _stack_item_0 = _tos_cache0; - top = _stack_item_0; - bottom = stack_pointer[-2]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache2 = top; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); } - - case _SWAP_3_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - top = _stack_item_1; - bottom = stack_pointer[-1]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache2 = top; - _tos_cache1 = _stack_item_0; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + l = left; + r = right; } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); + } - case _SWAP_3_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - top = _stack_item_2; - bottom = _stack_item_0; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache2 = top; - _tos_cache1 = _stack_item_1; - _tos_cache0 = bottom; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res, l, r)) { + if (PyJitRef_IsUnique(left)) { + ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE, 0, 0); + l = PyJitRef_Borrow(sym_new_null(ctx)); + r = right; } - - case _SWAP_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef top; - _PyStackRef bottom; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - top = _stack_item_0; - bottom = stack_pointer[-1 - (oparg-2)]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - _tos_cache0 = top; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer[-1 - (oparg-2)] = bottom; - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (PyJitRef_IsUnique(right)) { + ADD_OP(_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, 0, 0); + l = left; + r = PyJitRef_Borrow(sym_new_null(ctx)); } - - /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ - - /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ - - case _GUARD_IS_TRUE_POP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - flag = stack_pointer[-1]; - int is_true = PyStackRef_IsTrue(flag); - if (!is_true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + l = left; + r = right; } + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); + } - case _GUARD_IS_TRUE_POP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - flag = _stack_item_0; - int is_true = PyStackRef_IsTrue(flag); - if (!is_true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_BINARY_OP_ADD_UNICODE, (left, right -- res, l, r)) { + res = sym_new_type(ctx, &PyUnicode_Type); + l = left; + r = right; + } - case _GUARD_IS_TRUE_POP_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - flag = _stack_item_1; - int is_true = PyStackRef_IsTrue(flag); - if (!is_true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + op(_BINARY_OP_EXTEND, (descr/4, left, right -- res, l, r)) { + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr *)descr; + if (d != NULL && d->result_type != NULL) { + res = sym_new_type(ctx, d->result_type); + if (d->result_unique) { + res = PyJitRef_MakeUnique(res); } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_IS_TRUE_POP_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - flag = _stack_item_2; - int is_true = PyStackRef_IsTrue(flag); - if (!is_true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + res = sym_new_not_null(ctx); } + l = left; + r = right; + } - case _GUARD_IS_FALSE_POP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - flag = stack_pointer[-1]; - int is_false = PyStackRef_IsFalse(flag); - if (!is_false) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); + op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- res)) { + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } + GETLOCAL(this_instr->operand0) = sym_new_null(ctx); + } + + op(_BINARY_OP_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) { + getitem = sym_new_not_null(ctx); + PyTypeObject *tp = sym_get_type(container); + if (tp == NULL) { + PyObject *c = sym_get_probable_value(container); + if (c != NULL) { + tp = Py_TYPE(c); + } + } + if (tp != NULL) { + PyObject *getitem_o = ((PyHeapTypeObject *)tp)->_spec_cache.getitem; + sym_set_recorded_value(getitem, getitem_o); + } + } + + op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { + _Py_UOpsAbstractFrame *f = frame_new_from_symbol(ctx, getitem, NULL, 0); + if (f == NULL) { + break; + } + f->locals[0] = container; + f->locals[1] = sub; + new_frame = PyJitRef_WrapInvalid(f); + } + + op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res, s, i)) { + res = sym_new_type(ctx, &PyUnicode_Type); + s = str_st; + i = sub_st; + } + + op(_BINARY_OP_SUBSCR_USTR_INT, (str_st, sub_st -- res, s, i)) { + res = sym_new_type(ctx, &PyUnicode_Type); + s = str_st; + i = sub_st; + } + + op(_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, (tuple_st, sub_st -- tuple_st, sub_st)) { + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(index >= 0); + Py_ssize_t tuple_length = sym_tuple_length(tuple_st); + if (tuple_length != -1 && index < tuple_length) { + ADD_OP(_NOP, 0, 0); + } + } + } + + op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res, ts, ss)) { + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(index >= 0); + Py_ssize_t tuple_length = sym_tuple_length(tuple_st); + if (tuple_length == -1) { + // Unknown length + res = sym_new_not_null(ctx); } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IS_FALSE_POP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - flag = _stack_item_0; - int is_false = PyStackRef_IsFalse(flag); - if (!is_false) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + else { + assert(index < tuple_length); + res = sym_tuple_getitem(ctx, tuple_st, index); } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IS_FALSE_POP_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - flag = _stack_item_1; - int is_false = PyStackRef_IsFalse(flag); - if (!is_false) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + } + else { + res = sym_new_not_null(ctx); + } + ts = tuple_st; + ss = sub_st; + } + + op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res, ds, ss)) { + res = sym_new_not_null(ctx); + ds = dict_st; + ss = sub_st; + PyObject *sub = sym_get_const(ctx, sub_st); + if (sym_is_not_container(sub_st) && + sym_matches_type(dict_st, &PyFrozenDict_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(dict_st, sub_st, res); + } + else if (sub != NULL) { + optimize_dict_known_hash(ctx, dependencies, this_instr, + sub, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH); + } + } + + op(_BINARY_OP_SUBSCR_LIST_SLICE, (list_st, sub_st -- res, ls, ss)) { + res = sym_new_type(ctx, &PyList_Type); + ls = list_st; + ss = sub_st; + } + + op(_TO_BOOL, (value -- res)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, + _POP_TOP, _NOP); + if (!already_bool) { + res = sym_new_truthiness(ctx, value, true); + } + } + + op(_TO_BOOL_BOOL, (value -- value)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &value, + _POP_TOP, _NOP); + if (!already_bool) { + sym_set_type(value, &PyBool_Type); + } + } + + op(_TO_BOOL_INT, (value -- res, v)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, + _NOP, _SWAP); + if (!already_bool) { + sym_set_type(value, &PyLong_Type); + res = sym_new_truthiness(ctx, value, true); + } + v = value; + } + + op(_TO_BOOL_LIST, (value -- res, v)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, + _NOP, _SWAP); + if (!already_bool) { + res = sym_new_type(ctx, &PyBool_Type); + } + v = value; + } + + op(_TO_BOOL_NONE, (value -- res)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, + _POP_TOP, _NOP); + if (!already_bool) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); + } + } + + op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyUnicode_Type)) { + ADD_OP(_NOP, 0, 0); + } + sym_set_type(nos, &PyUnicode_Type); + } + + op(_GUARD_NOS_COMPACT_ASCII, (nos, unused -- nos, unused)) { + sym_set_type(nos, &PyUnicode_Type); + } + + op(_GUARD_TOS_UNICODE, (value -- value)) { + if (sym_matches_type(value, &PyUnicode_Type)) { + ADD_OP(_NOP, 0, 0); + } + sym_set_type(value, &PyUnicode_Type); + } + + op(_TO_BOOL_STR, (value -- res, v)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, + _NOP, _SWAP); + v = value; + if (!already_bool) { + res = sym_new_truthiness(ctx, value, true); + } + } + + op(_UNARY_NOT, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value, res); + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); + } + + op(_UNARY_NEGATIVE, (value -- res, v)) { + v = value; + REPLACE_OPCODE_IF_EVALUATES_PURE(value, res); + if (sym_matches_type(value, &PyFloat_Type) && PyJitRef_IsUnique(value)) { + ADD_OP(_UNARY_NEGATIVE_FLOAT_INPLACE, 0, 0); + v = PyJitRef_Borrow(sym_new_null(ctx)); + res = PyJitRef_MakeUnique(sym_new_type(ctx, &PyFloat_Type)); + } + else if (sym_is_compact_int(value)) { + res = sym_new_compact_int(ctx); + } + else { + PyTypeObject *type = sym_get_type(value); + if (type == &PyLong_Type || type == &PyFloat_Type) { + res = sym_new_type(ctx, type); } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IS_FALSE_POP_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - flag = _stack_item_2; - int is_false = PyStackRef_IsFalse(flag); - if (!is_false) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + else { + res = sym_new_not_null(ctx); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } + } - case _GUARD_BIT_IS_SET_POP_4_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_UNARY_INVERT, (value -- res, v)) { + v = value; + // Required to avoid a warning due to the deprecation of bitwise inversion of bools + if (!sym_matches_type(value, &PyBool_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value, res); } - - case _GUARD_BIT_IS_SET_POP_4_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + if (sym_matches_type(value, &PyLong_Type)) { + res = sym_new_type(ctx, &PyLong_Type); } - - case _GUARD_BIT_IS_SET_POP_4_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + res = sym_new_not_null(ctx); } + } - case _GUARD_BIT_IS_SET_POP_4_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_COMPARE_OP, (left, right -- res)) { + // Comparison between bytes and str or int is not impacted by this optimization as bytes + // is not a safe type (due to its ability to raise a warning during comparisons). + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + if (oparg & 16) { + res = sym_new_type(ctx, &PyBool_Type); + } + else { + res = _Py_uop_sym_new_not_null(ctx); + } + } + + op(_COMPARE_OP_INT, (left, right -- res, l, r)) { + int cmp_mask = oparg & (COMPARE_LT_MASK | COMPARE_GT_MASK | COMPARE_EQ_MASK); + + if (cmp_mask == COMPARE_EQ_MASK) { + res = sym_new_predicate(ctx, left, right, JIT_PRED_EQ); + } + else if (cmp_mask == (COMPARE_LT_MASK | COMPARE_GT_MASK)) { + res = sym_new_predicate(ctx, left, right, JIT_PRED_NE); + } + else { + res = sym_new_type(ctx, &PyBool_Type); + } + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } + + op(_COMPARE_OP_FLOAT, (left, right -- res, l, r)) { + int cmp_mask = oparg & (COMPARE_LT_MASK | COMPARE_GT_MASK | COMPARE_EQ_MASK); + + if (cmp_mask == COMPARE_EQ_MASK) { + res = sym_new_predicate(ctx, left, right, JIT_PRED_EQ); + } + else if (cmp_mask == (COMPARE_LT_MASK | COMPARE_GT_MASK)) { + res = sym_new_predicate(ctx, left, right, JIT_PRED_NE); + } + else { + res = sym_new_type(ctx, &PyBool_Type); + } + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } + + op(_COMPARE_OP_STR, (left, right -- res, l, r)) { + /* Cannot use predicate optimization here, as `a == b` + * does not imply that `a` is equivalent to `b`. `a` may be + * mortal, while `b` is immortal */ + res = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } - case _GUARD_BIT_IS_SET_POP_5_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_IS_OP, (left, right -- b, l, r)) { + b = sym_new_predicate(ctx, left, right, (oparg ? JIT_PRED_IS_NOT : JIT_PRED_IS)); + l = left; + r = right; + } + + op(_IS_NONE, (value -- b)) { + if (sym_is_const(ctx, value)) { + PyObject *value_o = sym_get_const(ctx, value); + b = sym_new_const(ctx, Py_IsNone(value_o) ? Py_True : Py_False); + } + else { + b = sym_new_type(ctx, &PyBool_Type); + } + } + + op(_CONTAINS_OP, (left, right -- b, l, r)) { + b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, b); + } + + op(_CONTAINS_OP_SET, (left, right -- b, l, r)) { + b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; + if (sym_is_not_container(left) && + sym_matches_type(right, &PyFrozenSet_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, b); + } + } + + op(_CONTAINS_OP_DICT, (left, right -- b, l, r)) { + b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; + } + + op(_LOAD_CONST, (-- value)) { + PyCodeObject *co = get_current_code_object(ctx); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); + } + + op(_LOAD_COMMON_CONSTANT, (-- value)) { + assert(oparg < NUM_COMMON_CONSTANTS); + PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); + } + + op(_LOAD_SMALL_INT, (-- value)) { + PyObject *val = PyLong_FromLong(oparg); + assert(val); + assert(_Py_IsImmortal(val)); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); + } + + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + } + + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_TOP_OPARG, (args[oparg] --)) { + for (int i = oparg-1; i >= 0; i--) { + optimize_pop_top(ctx, this_instr, args[i]); + } + } + + op(_POP_TOP, (value -- )) { + optimize_pop_top(ctx, this_instr, value); + } + + op(_POP_TOP_INT, (value --)) { + if (PyJitRef_IsBorrowed(value)) { + ADD_OP(_POP_TOP_NOP, 0, 0); + } + } + + op(_POP_TOP_FLOAT, (value --)) { + if (PyJitRef_IsBorrowed(value)) { + ADD_OP(_POP_TOP_NOP, 0, 0); + } + } + + op(_POP_TOP_UNICODE, (value --)) { + if (PyJitRef_IsBorrowed(value)) { + ADD_OP(_POP_TOP_NOP, 0, 0); + } + } + + op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + assert(oparg > 0); + bottom = PyJitRef_RemoveUnique(bottom); + top = bottom; + } + + op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { + JitOptRef temp = bottom; + bottom = top; + top = temp; + assert(oparg >= 2); + } + + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, o)) { + attr = sym_new_not_null(ctx); + (void)offset; + o = owner; + } + + op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr, o)) { + (void)dict_version; + (void)index; + attr = PyJitRef_NULL; + if (sym_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); + if (PyModule_CheckExact(mod)) { + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + PyObject *res = convert_global_to_const(this_instr, dict); + if (res == NULL) { + attr = sym_new_not_null(ctx); + } + else { + bool immortal = _Py_IsImmortal(res); + ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, + 0, (uintptr_t)res); + ADD_OP(_SWAP, 2, 0); + attr = sym_new_const(ctx, res); + } - case _GUARD_BIT_IS_SET_POP_5_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + } } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } + if (PyJitRef_IsNull(attr)) { + /* No conversion made. We don't know what `attr` is. */ + attr = sym_new_not_null(ctx); + } + o = owner; + } + + op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { + if (oparg & 1) { + ADD_OP(_PUSH_NULL, 0, 0); + null[0] = sym_new_null(ctx); + } + else { + ADD_OP(_NOP, 0, 0); + } + } + + op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { + (void)owner; + attr = sym_new_not_null(ctx); + if (oparg & 1) { + self_or_null[0] = sym_new_unknown(ctx); + } + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, o)) { + attr = sym_new_not_null(ctx); + (void)hint; + o = owner; + } + + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, o)) { + attr = sym_new_not_null(ctx); + (void)index; + o = owner; + } + + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { + (void)descr; + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, dependencies, this_instr, type, name, + _POP_TOP, _NOP); + } + + op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, dependencies, this_instr, type, name, + _POP_TOP, _NOP); + } + + op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, dependencies, this_instr, type, name, + _POP_TOP, _NOP); + } + + op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, dependencies, this_instr, type, name, + _NOP, _SWAP); + self = owner; + } + + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, dependencies, this_instr, type, name, + _NOP, _SWAP); + self = owner; + } + + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, dependencies, this_instr, type, name, + _NOP, _SWAP); + self = owner; + } + + op(_GUARD_LOAD_SUPER_ATTR_METHOD, (global_super_st, class_st, unused -- global_super_st, class_st, unused)) { + if (sym_get_const(ctx, global_super_st) == (PyObject *)&PySuper_Type) { + PyTypeObject *probable = (PyTypeObject *)sym_get_probable_value(class_st); + PyTypeObject *known = (PyTypeObject *)sym_get_const(ctx, class_st); + // not known, but has a probable type, promote the probable type + if (known == NULL && probable != NULL && PyType_Check(probable)) { + ADD_OP(_GUARD_NOS_TYPE_VERSION, 0, probable->tp_version_tag); + known = probable; + } + sym_set_const(class_st, (PyObject *)known); + } + else { + sym_set_const(global_super_st, (PyObject *)&PySuper_Type); + sym_set_type(class_st, &PyType_Type); + } + } + + op(_LOAD_SUPER_ATTR_METHOD, (global_super_st, class_st, self_st -- attr, self_or_null)) { + self_or_null = self_st; + PyTypeObject *su_type = (PyTypeObject *)sym_get_const(ctx, class_st); + PyTypeObject *obj_type = sym_get_type(self_st); + PyObject *name = get_co_name(ctx, oparg >> 2); + attr = lookup_super_attr(ctx, dependencies, this_instr, + su_type, obj_type, name, + _LOAD_CONST_INLINE_BORROW, + _LOAD_CONST_INLINE, _SWAP); + } + + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { + // + 1 for _SAVE_RETURN_OFFSET + // FIX ME -- This needs a version check and function watcher + PyCodeObject *co = (PyCodeObject *)((PyFunctionObject *)fget)->func_code; + _Py_UOpsAbstractFrame *f = frame_new(ctx, co, NULL, 0); + if (f == NULL) { + break; + } + f->locals[0] = owner; + new_frame = PyJitRef_WrapInvalid(f); + } + + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + } + + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + if (sym_get_func_version(callable) == func_version) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_func_version(ctx, callable, func_version); + } + } + + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { + PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); + assert(PyMethod_Check(method)); + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + uop_buffer_last(&ctx->out_buffer)->operand1 = (uintptr_t)method->im_func; + } + sym_set_type(callable, &PyMethod_Type); + } + + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(ctx, callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); + PyCodeObject *co = (PyCodeObject *)func->func_code; + if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { + ADD_OP(_NOP, 0 ,0); + } + } + } + } + + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { + int argcount = oparg; + assert(!PyJitRef_IsNull(self_or_null)); + assert(args != NULL); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, args, argcount)); + } else { + new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); + } + } + + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + (void)args; + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + } + + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { + new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); + } + + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); + } + + op(_PY_FRAME_EX, (func_st, null, callargs_st, kwargs_st -- ex_frame)) { + ex_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, func_st, NULL, 0)); + } + + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + (void)args; + PyObject *probable_callable = sym_get_probable_value(callable); + assert(probable_callable != NULL); + assert(PyType_Check(probable_callable)); + PyTypeObject *tp = (PyTypeObject *)probable_callable; + if (tp->tp_version_tag == type_version) { + // If the type version has not changed since we last saw it, + // then we know this __init__ is definitely the same one as in the cache. + // We can promote callable to a known constant. This does not need a + // type watcher, as we do not remove this _CHECK_AND_ALLOCATE_OBJECT guard. + // TODO: split up _CHECK_AND_ALLOCATE_OBJECT to the check then alloate, so we can + // eliminate the check. + PyHeapTypeObject *cls = (PyHeapTypeObject *)probable_callable; + PyObject *init = cls->_spec_cache.init; + assert(init != NULL); + assert(PyFunction_Check(init)); + callable = sym_new_const(ctx, init); + } + else { + callable = sym_new_not_null(ctx); + } + self_or_null = sym_new_not_null(ctx); + } + + op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, NULL, 0); + if (shim == NULL) { + break; + } + /* Push self onto stack of shim */ + shim->stack_pointer[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, oparg+1)); + } + + op(_RETURN_VALUE, (retval -- res)) { + JitOptRef temp = retval; + DEAD(retval); + SAVE_STACK(); + ctx->frame->stack_pointer = stack_pointer; + assert(this_instr[1].opcode == _RECORD_CODE); + PyCodeObject *returning_code = (PyCodeObject *)this_instr[1].operand0; + assert(PyCode_Check(returning_code)); + if (returning_code == NULL) { + ctx->done = true; + break; + } + if (frame_pop(ctx, returning_code)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + + RELOAD_STACK(); + res = temp; + } + + op(_RETURN_GENERATOR, ( -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + assert(this_instr[1].opcode == _RECORD_CODE); + PyCodeObject *returning_code = (PyCodeObject *)this_instr[1].operand0; + if (returning_code == NULL) { + ctx->done = true; + break; + } + assert(PyCode_Check(returning_code)); + if (frame_pop(ctx, returning_code)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + } + + op(_YIELD_VALUE, (retval -- value)) { + JitOptRef temp = retval; + DEAD(retval); + SAVE_STACK(); + ctx->frame->stack_pointer = stack_pointer; + assert(this_instr[1].opcode == _RECORD_CODE); + PyCodeObject *returning_code = (PyCodeObject *)this_instr[1].operand0; + if (returning_code == NULL) { + ctx->done = true; + break; + } + assert(PyCode_Check(returning_code)); + if (frame_pop(ctx, returning_code)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + RELOAD_STACK(); + value = temp; + } - case _GUARD_BIT_IS_SET_POP_5_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GET_ITER, (iterable -- iter, index_or_null)) { + if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { + iter = iterable; + index_or_null = sym_new_not_null(ctx); } - - case _GUARD_BIT_IS_SET_POP_5_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + iter = sym_new_not_null(ctx); + index_or_null = sym_new_unknown(ctx); } + } - case _GUARD_BIT_IS_SET_POP_6_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + op(_FOR_ITER_GEN_FRAME, (iter, unused -- iter, unused, gen_frame)) { + _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, iter, NULL, 0); + if (new_frame == NULL) { + ctx->done = true; break; } + new_frame->stack_pointer[0] = sym_new_const(ctx, Py_None); + new_frame->stack_pointer++; + gen_frame = PyJitRef_WrapInvalid(new_frame); + } - case _GUARD_BIT_IS_SET_POP_6_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { + _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, receiver, NULL, 0); + if (new_frame == NULL) { + ctx->done = true; break; } + new_frame->stack_pointer[0] = PyJitRef_StripReferenceInfo(v); + new_frame->stack_pointer++; + gen_frame = PyJitRef_WrapInvalid(new_frame); + } - case _GUARD_BIT_IS_SET_POP_6_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyCodeObject *co = sym_get_probable_func_code(callable); + if (co == NULL) { + ctx->done = true; break; } + ADD_OP(_CHECK_STACK_SPACE_OPERAND, 0, co->co_framesize); + } - case _GUARD_BIT_IS_SET_POP_6_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { + (void)framesize; + } - case _GUARD_BIT_IS_SET_POP_7_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_PUSH_FRAME, (new_frame -- )) { + SYNC_SP(); + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; } + ctx->frame->caller = true; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); + ctx->curr_frame_depth++; + stack_pointer = ctx->frame->stack_pointer; + assert(ctx->frame->locals != NULL); + } - case _GUARD_BIT_IS_SET_POP_7_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { + (void)top; + /* This has to be done manually */ + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); + } + } + + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8], top[0])) { + (void)top; + /* This has to be done manually */ + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); + } + } + + op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { + if (sym_matches_type(iter, &PyTuple_Type)) { + ADD_OP(_NOP, 0, 0); + } + sym_set_type(iter, &PyTuple_Type); + } + + op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { + if (sym_matches_type(iter, &PyList_Type)) { + ADD_OP(_NOP, 0, 0); + } + else { + sym_set_type(iter, &PyList_Type); } - - case _GUARD_BIT_IS_SET_POP_7_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + } + + op(_ITER_CHECK_RANGE, (iter, null_or_index -- iter, null_or_index)) { + if (sym_matches_type(iter, &PyRange_Type)) { + ADD_OP(_NOP, 0, 0); + } + else { + sym_set_type(iter, &PyRange_Type); + } + } + + op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { + next = sym_new_type(ctx, &PyLong_Type); + } + + op(_CALL_TYPE_1, (unused, unused, arg -- res, a)) { + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + ADD_OP(_SWAP, 3, 0); + ADD_OP(_POP_TOP, 0, 0); + ADD_OP(_POP_TOP, 0, 0); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)type); + ADD_OP(_SWAP, 2, 0); + } + else { + res = sym_new_not_null(ctx); + } + a = arg; + } + + op(_CALL_STR_1, (unused, unused, arg -- res, a)) { + if (sym_matches_type(arg, &PyUnicode_Type)) { + // e.g. str('foo') or str(foo) where foo is known to be a string + // Note: we must strip the reference information because it goes + // through str() which strips the reference information from it. + res = PyJitRef_StripReferenceInfo(arg); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } + a = arg; + } + + op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { + // the result is always a bool, but sometimes we can + // narrow it down to True or False + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + // isinstance(inst, cls) where both inst and cls have + // known types, meaning we can deduce either True or False + + // The below check is equivalent to PyObject_TypeCheck(inst, cls) + PyObject *out = Py_False; + if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { + out = Py_True; + } + sym_set_const(res, out); + ADD_OP(_POP_TOP, 0, 0); + ADD_OP(_POP_TOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP, 0, 0); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + } + } + + op(_CALL_LIST_APPEND, (callable, self, arg -- none, c, s)) { + (void)(arg); + c = callable; + s = self; + none = sym_new_const(ctx, Py_None); + } + + op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyType_Type)) { + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_vectorcall != NULL) { + ADD_OP(_NOP, 0, 0); } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_BIT_IS_SET_POP_7_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyType_Type); } + } - case _GUARD_BIT_IS_SET_POP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = CURRENT_OPARG(); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); + op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BIT_IS_SET_POP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { + ADD_OP(_NOP, 0, 0); } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_BIT_IS_SET_POP_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyCFunction_Type); } + } - case _GUARD_BIT_IS_SET_POP_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { + if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { + ADD_OP(_NOP, 0, 0); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_BIT_IS_UNSET_POP_4_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyCFunction_Type); } + } - case _GUARD_BIT_IS_UNSET_POP_4_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { + if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { + ADD_OP(_NOP, 0, 0); } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_BIT_IS_UNSET_POP_4_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyCFunction_Type); } + } - case _GUARD_BIT_IS_UNSET_POP_4_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 4; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + callable = sym_new_not_null(ctx); + } - case _GUARD_BIT_IS_UNSET_POP_5_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { + res = sym_new_not_null(ctx); + c = callable; + if (sym_is_not_null(self_or_null)) { + args--; + s = args[0]; } - - case _GUARD_BIT_IS_UNSET_POP_5_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (sym_is_null(self_or_null)) { + s = args[0]; } - - case _GUARD_BIT_IS_UNSET_POP_5_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + s = sym_new_unknown(ctx); } + } - case _GUARD_BIT_IS_UNSET_POP_5_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 5; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + callable = sym_new_not_null(ctx); + } - case _GUARD_BIT_IS_UNSET_POP_6_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + callable = sym_new_not_null(ctx); + } - case _GUARD_BIT_IS_UNSET_POP_6_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BIT_IS_UNSET_POP_6_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BIT_IS_UNSET_POP_6_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 6; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + else { + self_type = sym_get_type(args[0]); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BIT_IS_UNSET_POP_7_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args == 2 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_BIT_IS_UNSET_POP_7_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyMethodDescr_Type); } + } - case _GUARD_BIT_IS_UNSET_POP_7_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BIT_IS_UNSET_POP_7_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = 7; - assert(oparg == CURRENT_OPARG()); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BIT_IS_UNSET_POP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - oparg = CURRENT_OPARG(); - flag = stack_pointer[-1]; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); + else { + self_type = sym_get_type(args[0]); } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_BIT_IS_UNSET_POP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - flag = _stack_item_0; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args != 0 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_BIT_IS_UNSET_POP_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - flag = _stack_item_1; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyMethodDescr_Type); } + } - case _GUARD_BIT_IS_UNSET_POP_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef flag; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - flag = _stack_item_2; - #ifdef Py_STACKREF_DEBUG - uintptr_t bits = flag.index; - #else - uintptr_t bits = flag.bits; - #endif - uintptr_t set = (1 << oparg) & bits; - if (set != 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IS_NONE_POP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef val; - val = stack_pointer[-1]; - int is_none = PyStackRef_IsNone(val); - if (!is_none) { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); } - SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IS_NONE_POP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef val; - _PyStackRef _stack_item_0 = _tos_cache0; - val = _stack_item_0; - int is_none = PyStackRef_IsNone(val); - if (!is_none) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } + else { + self_type = sym_get_type(args[0]); } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IS_NONE_POP_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef val; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - val = _stack_item_1; - int is_none = PyStackRef_IsNone(val); - if (!is_none) { - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args == 1 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_IS_NONE_POP_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef val; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - val = _stack_item_2; - int is_none = PyStackRef_IsNone(val); - if (!is_none) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyMethodDescr_Type); } + } - case _GUARD_IS_NOT_NONE_POP_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef val; - _PyStackRef _stack_item_0 = _tos_cache0; - val = _stack_item_0; - int is_none = PyStackRef_IsNone(val); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_none) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_CHECK_RECURSION_LIMIT, ( -- )) { + if (ctx->frame->is_c_recursion_checked) { + ADD_OP(_NOP, 0, 0); } + ctx->frame->is_c_recursion_checked = true; + } - case _JUMP_TO_TOP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - JUMP_TO_JUMP_TARGET(); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res, c, s)) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); } - - case _SET_IP_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); - frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + res = sym_new_not_null(ctx); + c = callable; + if (sym_is_not_null(self_or_null)) { + args--; + s = args[0]; } - - case _SET_IP_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); - frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (sym_is_null(self_or_null)) { + s = args[0]; } - - case _SET_IP_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); - frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + s = sym_new_unknown(ctx); } + } - case _SET_IP_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); - frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); } + callable = sym_new_not_null(ctx); + } - case _CHECK_STACK_SPACE_OPERAND_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); - assert(framesize <= INT_MAX); - if (!_PyThreadState_HasStackSpace(tstate, framesize)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); } + callable = sym_new_not_null(ctx); + } - case _CHECK_STACK_SPACE_OPERAND_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); - assert(framesize <= INT_MAX); - if (!_PyThreadState_HasStackSpace(tstate, framesize)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && + (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { + int total_args = oparg; + if (sym_is_not_null(self_or_null)) { + total_args++; } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_STACK_SPACE_OPERAND_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); - assert(framesize <= INT_MAX); - if (!_PyThreadState_HasStackSpace(tstate, framesize)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + PyTypeObject *self_type = NULL; + if (sym_is_not_null(self_or_null)) { + self_type = sym_get_type(self_or_null); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _CHECK_STACK_SPACE_OPERAND_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); - assert(framesize <= INT_MAX); - if (!_PyThreadState_HasStackSpace(tstate, framesize)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); + else { + self_type = sym_get_type(args[0]); + } + PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + if (total_args != 0 && + ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && + self_type == d_type) { + ADD_OP(_NOP, 0, 0); } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _SAVE_RETURN_OFFSET_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(callable, &PyMethodDescr_Type); } + } - case _SAVE_RETURN_OFFSET_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + PyObject *callable_o = sym_get_const(ctx, callable); + if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) + && sym_is_not_null(self_or_null)) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyCFunction cfunc = method->d_method->ml_meth; + ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); } - - case _SAVE_RETURN_OFFSET_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + res = sym_new_not_null(ctx); + c = callable; + if (sym_is_not_null(self_or_null)) { + args--; + s = args[0]; + a = args[1]; } - - case _SAVE_RETURN_OFFSET_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + s = sym_new_unknown(ctx); + a = sym_new_unknown(ctx); } + } - case _EXIT_TRACE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - _PyExitData *exit = (_PyExitData *)exit_p; - #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) - + exit->target; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("SIDE EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - tstate->jit_exit = exit; - SET_CURRENT_CACHED_VALUES(0); - TIER2_TO_TIER2(exit->executor); - } + op(_CALL_INTRINSIC_1, (value -- res, v)) { + res = sym_new_not_null(ctx); + v = value; + } + + op(_CALL_INTRINSIC_2, (value2_st, value1_st -- res, vs1, vs2)) { + res = sym_new_not_null(ctx); + vs1 = value1_st; + vs2 = value2_st; + } + + op(_GUARD_IS_TRUE_POP, (flag -- )) { + sym_apply_predicate_narrowing(ctx, flag, true); + + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, ctx, value != Py_True); + } + else { + int bit = get_test_bit_for_bools(); + if (bit) { + REPLACE_OP(this_instr, + test_bit_set_in_true(bit) ? + _GUARD_BIT_IS_SET_POP : + _GUARD_BIT_IS_UNSET_POP, bit, 0); + } + } + sym_set_const(flag, Py_True); + } + + op(_GUARD_IS_FALSE_POP, (flag -- )) { + sym_apply_predicate_narrowing(ctx, flag, false); + + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, ctx, value != Py_False); + } + else { + int bit = get_test_bit_for_bools(); + if (bit) { + REPLACE_OP(this_instr, + test_bit_set_in_true(bit) ? + _GUARD_BIT_IS_UNSET_POP : + _GUARD_BIT_IS_SET_POP, bit, 0); + } + } + sym_set_const(flag, Py_False); + } + + op(_GUARD_IS_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); + assert(value != NULL); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); + } + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); + eliminate_pop_guard(this_instr, ctx, true); + } + sym_set_const(val, Py_None); + } + + op(_GUARD_IS_NOT_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); + assert(value != NULL); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); + } + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); + eliminate_pop_guard(this_instr, ctx, false); + } + } + + op(_CHECK_PEP_523, (--)) { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + ADD_OP(_NOP, 0 ,0); + } + } + + op(_INSERT_NULL, (self -- method_and_self[2])) { + method_and_self[0] = sym_new_null(ctx); + method_and_self[1] = self; + } + + op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { + method_and_self[0] = sym_new_not_null(ctx); + method_and_self[1] = sym_new_unknown(ctx); + } + + op(_JUMP_TO_TOP, (--)) { + ctx->done = true; + } + + op(_EXIT_TRACE, (exit_p/4 --)) { + (void)exit_p; + ctx->done = true; + } + + op(_DEOPT, (--)) { + ctx->done = true; + } + + op(_REPLACE_WITH_TRUE, (value -- res, v)) { + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_SWAP, 2, 0); + res = sym_new_const(ctx, Py_True); + v = value; + } - case _EXIT_TRACE_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - _PyExitData *exit = (_PyExitData *)exit_p; - #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) - + exit->target; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("SIDE EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - } - #endif - tstate->jit_exit = exit; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - TIER2_TO_TIER2(exit->executor); - } + op(_BUILD_TUPLE, (values[oparg] -- tup)) { + tup = sym_new_tuple(ctx, oparg, values); + tup = PyJitRef_MakeUnique(tup); + } - case _EXIT_TRACE_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - _PyExitData *exit = (_PyExitData *)exit_p; - #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) - + exit->target; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("SIDE EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - } - #endif - tstate->jit_exit = exit; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - TIER2_TO_TIER2(exit->executor); - } + op(_BUILD_LIST, (values[oparg] -- list)) { + list = sym_new_type(ctx, &PyList_Type); + } - case _EXIT_TRACE_r30: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - _PyExitData *exit = (_PyExitData *)exit_p; - #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) - + exit->target; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("SIDE EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - } - #endif - tstate->jit_exit = exit; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - TIER2_TO_TIER2(exit->executor); - } + op(_BUILD_SLICE, (args[oparg] -- slice)) { + slice = sym_new_type(ctx, &PySlice_Type); + } - case _DYNAMIC_EXIT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - #if defined(Py_DEBUG) && !defined(_Py_JIT) - _PyExitData *exit = (_PyExitData *)exit_p; - _Py_CODEUNIT *target = frame->instr_ptr; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("DYNAMIC EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(frame->instr_ptr); - } + op(_BUILD_MAP, (values[oparg*2] -- map)) { + map = sym_new_type(ctx, &PyDict_Type); + } - case _DYNAMIC_EXIT_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - #if defined(Py_DEBUG) && !defined(_Py_JIT) - _PyExitData *exit = (_PyExitData *)exit_p; - _Py_CODEUNIT *target = frame->instr_ptr; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("DYNAMIC EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - } - #endif - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - GOTO_TIER_ONE(frame->instr_ptr); - } + op(_BUILD_STRING, (pieces[oparg] -- str)) { + str = sym_new_type(ctx, &PyUnicode_Type); + } - case _DYNAMIC_EXIT_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - #if defined(Py_DEBUG) && !defined(_Py_JIT) - _PyExitData *exit = (_PyExitData *)exit_p; - _Py_CODEUNIT *target = frame->instr_ptr; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("DYNAMIC EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - } - #endif - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - GOTO_TIER_ONE(frame->instr_ptr); - } + op(_BUILD_SET, (values[oparg] -- set)) { + set = sym_new_type(ctx, &PySet_Type); + } - case _DYNAMIC_EXIT_r30: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); - #if defined(Py_DEBUG) && !defined(_Py_JIT) - _PyExitData *exit = (_PyExitData *)exit_p; - _Py_CODEUNIT *target = frame->instr_ptr; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 3) { - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("DYNAMIC EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - } - #endif - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - GOTO_TIER_ONE(frame->instr_ptr); - } + op(_SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1], i)) { + (void)set; + i = iterable; + } - case _CHECK_VALIDITY_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - if (!current_executor->vm_data.valid) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1], i)) { + (void)list_st; + i = iterable_st; + } - case _CHECK_VALIDITY_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - if (!current_executor->vm_data.valid) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1], u)) { + (void)callable; + (void)dict; + u = update; + } - case _CHECK_VALIDITY_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - if (!current_executor->vm_data.valid) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { + if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 2) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, oparg, 0); } + val0 = sym_tuple_getitem(ctx, seq, 0); + val1 = sym_tuple_getitem(ctx, seq, 1); + } - case _CHECK_VALIDITY_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - if (!current_executor->vm_data.valid) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { + if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == 3) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, oparg, 0); } - - case _LOAD_CONST_INLINE_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectNew(ptr); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (PyJitRef_IsUnique(seq) && sym_tuple_length(seq) == oparg) { + ADD_OP(_UNPACK_SEQUENCE_UNIQUE_TUPLE, oparg, 0); } - - case _LOAD_CONST_INLINE_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectNew(ptr); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + for (int i = 0; i < oparg; i++) { + values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } + } - case _LOAD_CONST_INLINE_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectNew(ptr); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_CALL_TUPLE_1, (callable, null, arg -- res, a)) { + if (sym_matches_type(arg, &PyTuple_Type)) { + // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple + // Note: we must strip the reference information because it goes + // through tuple() which strips the reference information from it. + res = PyJitRef_StripReferenceInfo(arg); } - - case _LOAD_CONST_INLINE_BORROW_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + res = sym_new_type(ctx, &PyTuple_Type); } + a = arg; + } - case _LOAD_CONST_INLINE_BORROW_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_TOS_LIST, (tos -- tos)) { + if (sym_matches_type(tos, &PyList_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(tos, &PyList_Type); } + } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - arg = stack_pointer[-1]; - callable = stack_pointer[-3]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyList_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - arg = _stack_item_0; - callable = stack_pointer[-2]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(nos, &PyList_Type); } + } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - arg = _stack_item_1; - callable = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_TOS_TUPLE, (tos -- tos)) { + if (sym_matches_type(tos, &PyTuple_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - arg = _stack_item_2; - callable = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(tos, &PyTuple_Type); } + } - case _START_EXECUTOR_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *executor = (PyObject *)CURRENT_OPERAND0_64(); - #ifndef _Py_JIT - assert(current_executor == (_PyExecutorObject*)executor); - #endif - assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); - tstate->current_executor = (PyObject *)current_executor; - if (!current_executor->vm_data.valid) { - assert(tstate->jit_exit->executor == current_executor); - assert(tstate->current_executor == executor); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyExecutor_ClearExit(tstate->jit_exit); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyTuple_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _MAKE_WARM_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - current_executor->vm_data.cold = false; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(nos, &PyTuple_Type); } + } - case _MAKE_WARM_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - current_executor->vm_data.cold = false; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyDict_Type)) { + ADD_OP(_NOP, 0, 0); } + sym_set_type(nos, &PyDict_Type); + } - case _MAKE_WARM_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - current_executor->vm_data.cold = false; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_NOS_ANY_DICT, (nos, unused -- nos, unused)) { + PyTypeObject *tp = sym_get_type(nos); + if (tp == &PyDict_Type || tp == &PyFrozenDict_Type) { + ADD_OP(_NOP, 0, 0); } + } - case _MAKE_WARM_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - current_executor->vm_data.cold = false; - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_TOS_ANY_DICT, (tos -- tos)) { + PyTypeObject *tp = sym_get_type(tos); + if (tp == &PyDict_Type || tp == &PyFrozenDict_Type) { + ADD_OP(_NOP, 0, 0); } - - case _FATAL_ERROR_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - assert(0); - Py_FatalError("Fatal error uop executed."); - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + // Narrowing the guard based on the probable type. + tp = sym_get_probable_type(tos); + if (tp == &PyDict_Type) { + ADD_OP(_GUARD_TOS_DICT, 0, 0); + } + else if (tp == &PyFrozenDict_Type) { + ADD_OP(_GUARD_TOS_FROZENDICT, 0, 0); + } } + } - case _FATAL_ERROR_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - assert(0); - Py_FatalError("Fatal error uop executed."); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_TOS_DICT, (tos -- tos)) { + if (sym_matches_type(tos, &PyDict_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _FATAL_ERROR_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - assert(0); - Py_FatalError("Fatal error uop executed."); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_type(tos, &PyDict_Type); } + } - case _FATAL_ERROR_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - assert(0); - Py_FatalError("Fatal error uop executed."); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_TOS_FROZENDICT, (tos -- tos)) { + if (sym_matches_type(tos, &PyFrozenDict_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _DEOPT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + else { + sym_set_type(tos, &PyFrozenDict_Type); } + } - case _DEOPT_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + op(_GUARD_TOS_ANY_SET, (tos -- tos)) { + PyTypeObject *tp = sym_get_type(tos); + if (tp == &PySet_Type || tp == &PyFrozenSet_Type) { + ADD_OP(_NOP, 0, 0); } - - case _DEOPT_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + else { + // Narrowing the guard based on the probable type. + tp = sym_get_probable_type(tos); + if (tp == &PySet_Type) { + ADD_OP(_GUARD_TOS_SET, 0, 0); + } + else if (tp == &PyFrozenSet_Type) { + ADD_OP(_GUARD_TOS_FROZENSET, 0, 0); + } } + } - case _DEOPT_r30: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + op(_GUARD_TOS_SET, (tos -- tos)) { + if (sym_matches_type(tos, &PySet_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _HANDLE_PENDING_AND_DEOPT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + else { + sym_set_type(tos, &PySet_Type); } + } - case _HANDLE_PENDING_AND_DEOPT_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + op(_GUARD_TOS_FROZENSET, (tos -- tos)) { + if (sym_matches_type(tos, &PyFrozenSet_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _HANDLE_PENDING_AND_DEOPT_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + else { + sym_set_type(tos, &PyFrozenSet_Type); } + } - case _HANDLE_PENDING_AND_DEOPT_r30: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + op(_GUARD_TOS_SLICE, (tos -- tos)) { + if (sym_matches_type(tos, &PySlice_Type)) { + ADD_OP(_NOP, 0, 0); } - - case _ERROR_POP_N_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - uint32_t target = (uint32_t)CURRENT_OPERAND0_32(); - assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(NULL); + else { + sym_set_type(tos, &PySlice_Type); } + } - case _SPILL_OR_RELOAD_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _tos_cache0 = stack_pointer[-1]; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { + if (sym_is_null(null)) { + ADD_OP(_NOP, 0, 0); } - - case _SPILL_OR_RELOAD_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = stack_pointer[-2]; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_null(null); } + } - case _SPILL_OR_RELOAD_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _tos_cache2 = stack_pointer[-1]; - _tos_cache1 = stack_pointer[-2]; - _tos_cache0 = stack_pointer[-3]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { + if (sym_is_not_null(nos)) { + ADD_OP(_NOP, 0, 0); } - - case _SPILL_OR_RELOAD_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_non_null(nos); } + } - case _SPILL_OR_RELOAD_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _tos_cache1 = _stack_item_0; - _tos_cache0 = stack_pointer[-1]; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { + if (sym_is_null(null)) { + ADD_OP(_NOP, 0, 0); } - - case _SPILL_OR_RELOAD_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _tos_cache2 = _stack_item_0; - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = stack_pointer[-2]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_null(null); } + } - case _SPILL_OR_RELOAD_r20: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + ADD_OP(_NOP, 0, 0); } - - case _SPILL_OR_RELOAD_r21: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _tos_cache0 = _stack_item_1; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_const(callable, (PyObject *)&PyType_Type); } + } - case _SPILL_OR_RELOAD_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _tos_cache2 = _stack_item_1; - _tos_cache1 = _stack_item_0; - _tos_cache0 = stack_pointer[-1]; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + ADD_OP(_NOP, 0, 0); } - - case _SPILL_OR_RELOAD_r30: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - SET_CURRENT_CACHED_VALUES(0); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer[2] = _stack_item_2; - stack_pointer += 3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_const(callable, (PyObject *)&PyTuple_Type); } + } - case _SPILL_OR_RELOAD_r31: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - _tos_cache0 = _stack_item_2; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer[0] = _stack_item_0; - stack_pointer[1] = _stack_item_1; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + ADD_OP(_NOP, 0, 0); } - - case _SPILL_OR_RELOAD_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - _tos_cache1 = _stack_item_2; - _tos_cache0 = _stack_item_1; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer[0] = _stack_item_0; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_const(callable, (PyObject *)&PyUnicode_Type); } + } - case _TIER2_RESUME_CHECK_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - if (eval_breaker != iversion) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } + op(_CALL_LEN, (callable, null, arg -- res, a, c)) { + res = sym_new_type(ctx, &PyLong_Type); + Py_ssize_t length = sym_tuple_length(arg); - case _TIER2_RESUME_CHECK_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - if (eval_breaker != iversion) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + // Not a tuple, check if it's a const string + if (length < 0 && sym_is_const(ctx, arg)) { + PyObject *const_val = sym_get_const(ctx, arg); + if (const_val != NULL) { + if (PyUnicode_CheckExact(const_val)) { + length = PyUnicode_GET_LENGTH(const_val); + } + else if (PyBytes_CheckExact(const_val)) { + length = PyBytes_GET_SIZE(const_val); + } } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - case _TIER2_RESUME_CHECK_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + if (length >= 0) { + PyObject *temp = PyLong_FromSsize_t(length); + if (temp == NULL) { + goto error; } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - if (eval_breaker != iversion) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + if (_Py_IsImmortal(temp)) { + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + res = sym_new_const(ctx, temp); + Py_DECREF(temp); } + a = arg; + c = callable; + } - case _TIER2_RESUME_CHECK_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - if (eval_breaker != iversion) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GET_LEN, (obj -- obj, len)) { + Py_ssize_t tuple_length = sym_tuple_length(obj); + if (tuple_length == -1) { + len = sym_new_type(ctx, &PyLong_Type); } - - case _COLD_EXIT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyExitData *exit = tstate->jit_exit; - assert(exit != NULL); - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; - _Py_BackoffCounter temperature = exit->temperature; - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - assert(tstate->jit_exit == exit); - exit->executor = executor; - SET_CURRENT_CACHED_VALUES(0); - TIER2_TO_TIER2(exit->executor); + else { + assert(tuple_length >= 0); + PyObject *temp = PyLong_FromSsize_t(tuple_length); + if (temp == NULL) { + goto error; } - else { - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, stack_pointer, chain_depth, exit, target->op.arg, previous_executor); - exit->temperature = restart_backoff_counter(exit->temperature); - if (succ) { - GOTO_TIER_ONE_CONTINUE_TRACING(target); - } - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(target); + if (_Py_IsImmortal(temp)) { + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } + len = sym_new_const(ctx, temp); + Py_DECREF(temp); } + } - case _COLD_DYNAMIC_EXIT_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _Py_CODEUNIT *target = frame->instr_ptr; - SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(target); + op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { + PyObject *len = _PyInterpreterState_GET()->callable_cache.len; + if (sym_get_const(ctx, callable) == len) { + ADD_OP(_NOP, 0, 0); } - - case _GUARD_CODE_VERSION__PUSH_FRAME_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_const(callable, len); } + } - case _GUARD_CODE_VERSION__PUSH_FRAME_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { + PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; + if (sym_get_const(ctx, callable) == isinstance) { + ADD_OP(_NOP, 0, 0); } - - case _GUARD_CODE_VERSION__PUSH_FRAME_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_const(callable, isinstance); } + } - case _GUARD_CODE_VERSION__PUSH_FRAME_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { + PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; + if (sym_get_const(ctx, callable) == list_append) { + ADD_OP(_NOP, 0, 0); } - - case _GUARD_CODE_VERSION_YIELD_VALUE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + sym_set_const(callable, list_append); } + } - case _GUARD_CODE_VERSION_YIELD_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_BINARY_SLICE, (container, start, stop -- res)) { + // Slicing a string/list/tuple always returns the same type. + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); } - - case _GUARD_CODE_VERSION_YIELD_VALUE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else { + res = sym_new_not_null(ctx); } + } - case _GUARD_CODE_VERSION_YIELD_VALUE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } + op(_GUARD_GLOBALS_VERSION, (version/1 --)) { + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CODE_VERSION_RETURN_VALUE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + /* Do nothing */ } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CODE_VERSION_RETURN_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; } - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CODE_VERSION_RETURN_VALUE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + if (ctx->frame->globals_checked_version == version) { + ADD_OP(_NOP, 0, 0); } } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } + ctx->frame->globals_checked_version = version; + } - case _GUARD_CODE_VERSION_RETURN_VALUE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_LOAD_GLOBAL_BUILTINS, (version/1, index/1 -- res)) { + (void)version; + (void)index; + PyObject *cnst = NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = interp->builtins; + if (incorrect_keys(builtins, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; } - - case _GUARD_CODE_VERSION_RETURN_GENERATOR_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + /* Do nothing */ } - - case _GUARD_CODE_VERSION_RETURN_GENERATOR_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } + else { + if (!ctx->builtins_watched) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + ctx->builtins_watched = true; } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_CODE_VERSION_RETURN_GENERATOR_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } + if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { + cnst = convert_global_to_const(this_instr, builtins); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_CODE_VERSION_RETURN_GENERATOR_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); - PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - assert(PyCode_Check(code)); - if (((PyCodeObject *)code)->co_version != version) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + if (cnst == NULL) { + res = sym_new_not_null(ctx); } - - case _GUARD_IP__PUSH_FRAME_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr; - if (target != (_Py_CODEUNIT *)ip) { - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } + else { + if (_Py_IsImmortal(cnst)) { + res = PyJitRef_Borrow(sym_new_const(ctx, cnst)); } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IP__PUSH_FRAME_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr; - if (target != (_Py_CODEUNIT *)ip) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } + else { + res = sym_new_const(ctx, cnst); } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } + } - case _GUARD_IP__PUSH_FRAME_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr; - if (target != (_Py_CODEUNIT *)ip) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } + op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res)) { + (void)index; + PyObject *cnst = NULL; + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IP__PUSH_FRAME_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr; - if (target != (_Py_CODEUNIT *)ip) { - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + /* Do nothing */ } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IP_YIELD_VALUE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; } - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IP_YIELD_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { + REPLACE_OP(uop_buffer_last(&ctx->out_buffer), _GUARD_GLOBALS_VERSION, 0, version); + ctx->frame->globals_checked_version = version; } - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_IP_YIELD_VALUE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + if (ctx->frame->globals_checked_version == version) { + cnst = convert_global_to_const(this_instr, globals); } } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } - - case _GUARD_IP_YIELD_VALUE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + if (cnst == NULL) { + res = sym_new_not_null(ctx); } + else { + if (_Py_IsImmortal(cnst)) { + res = PyJitRef_Borrow(sym_new_const(ctx, cnst)); + } + else { + res = sym_new_const(ctx, cnst); + } + } + } + + op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res, ls, ss)) { + res = sym_new_unknown(ctx); + ls = list_st; + ss = sub_st; + } + + op(_MAKE_FUNCTION, (codeobj_st -- func, co)) { + func = sym_new_type(ctx, &PyFunction_Type); + co = codeobj_st; + } + + op(_MATCH_CLASS, (subject, type, names -- attrs, s, tp, n)) { + attrs = sym_new_not_null(ctx); + s = subject; + tp = type; + n = names; + } - case _GUARD_IP_RETURN_VALUE_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } + op(_DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1], upd)) { + (void)dict; + upd = update; + } + + op(_RECORD_TOS, (tos -- tos)) { + sym_set_recorded_value(tos, (PyObject *)this_instr->operand0); + } + + op(_RECORD_TOS_TYPE, (tos -- tos)) { + PyTypeObject *tp = (PyTypeObject *)this_instr->operand0; + sym_set_recorded_type(tos, tp); + } + + op(_RECORD_NOS, (nos, tos -- nos, tos)) { + sym_set_recorded_value(nos, (PyObject *)this_instr->operand0); + } + + op(_RECORD_4OS, (value, _3os, nos, tos -- value, _3os, nos, tos)) { + sym_set_recorded_value(value, (PyObject *)this_instr->operand0); + } + + op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { + sym_set_recorded_value(func, (PyObject *)this_instr->operand0); + } + + op(_RECORD_NOS_GEN_FUNC, (nos, tos -- nos, tos)) { + PyFunctionObject *func = (PyFunctionObject *)this_instr->operand0; + assert(func == NULL || PyFunction_Check(func)); + sym_set_recorded_gen_func(nos, func); + } + + op(_RECORD_3OS_GEN_FUNC, (gen, nos, tos -- gen, nos, tos)) { + PyFunctionObject *func = (PyFunctionObject *)this_instr->operand0; + assert(func == NULL || PyFunction_Check(func)); + sym_set_recorded_gen_func(gen, func); + } + + op(_GUARD_CODE_VERSION__PUSH_FRAME, (version/2 -- )) { + PyCodeObject *co = get_current_code_object(ctx); + if (co->co_version == version) { + _Py_BloomFilter_Add(dependencies, co); + // Functions derive their version from code objects. + if (sym_get_func_version(ctx->frame->callable) == version) { + REPLACE_OP(this_instr, _NOP, 0, 0); } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; } + else { + ctx->done = true; + } + } - case _GUARD_IP_RETURN_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CODE_VERSION_RETURN_VALUE, (version/2 -- )) { + (void)version; + if (ctx->frame->caller) { + REPLACE_OP(this_instr, _NOP, 0, 0); } + } - case _GUARD_IP_RETURN_VALUE_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CODE_VERSION_YIELD_VALUE, (version/2 -- )) { + (void)version; + if (ctx->frame->caller) { + REPLACE_OP(this_instr, _NOP, 0, 0); } + } - case _GUARD_IP_RETURN_VALUE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_CODE_VERSION_RETURN_GENERATOR, (version/2 -- )) { + (void)version; + if (ctx->frame->caller) { + REPLACE_OP(this_instr, _NOP, 0, 0); } + } - case _GUARD_IP_RETURN_GENERATOR_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - } - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { + (void)ip; + stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); + if (sym_get_func_version(ctx->frame->callable) != 0 && + // We can remove this guard for simple function call targets. + (((PyCodeObject *)ctx->frame->func->func_code)->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == 0) { + REPLACE_OP(this_instr, _NOP, 0, 0); } + } - case _GUARD_IP_RETURN_GENERATOR_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { + (void)ip; + if (ctx->frame->caller) { + REPLACE_OP(this_instr, _NOP, 0, 0); } + stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); + } - case _GUARD_IP_RETURN_GENERATOR_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { + (void)ip; + if (ctx->frame->caller) { + REPLACE_OP(this_instr, _NOP, 0, 0); } + stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); + } - case _GUARD_IP_RETURN_GENERATOR_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); - _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; - if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; + op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { + (void)ip; + if (ctx->frame->caller) { + REPLACE_OP(this_instr, _NOP, 0, 0); } + stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); + } + - /* _TRACE_RECORD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ +// END BYTECODES // -#undef TIER_TWO +} From eb3ea3f7203578a2b5cd386ef9c28ab25d51abd8 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mishra Date: Sun, 12 Apr 2026 16:21:51 +0530 Subject: [PATCH 5/5] Move tests to Lib/test and simplify prev_uop tracking per review --- Lib/test/test_generated_cases.py | 5353 +++++++++++++++-------------- Tools/cases_generator/analyzer.py | 2910 ++++++++-------- 2 files changed, 4166 insertions(+), 4097 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 57991b5b6b859c..ce34aefaf72828 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1,2642 +1,2711 @@ -import contextlib -import os -import sys -import tempfile -import unittest - -from test import support -from test import test_tools - - -def skip_if_different_mount_drives(): - if sys.platform != "win32": - return - ROOT = os.path.dirname(os.path.dirname(__file__)) - root_drive = os.path.splitroot(ROOT)[0] - cwd_drive = os.path.splitroot(os.getcwd())[0] - if root_drive != cwd_drive: - # May raise ValueError if ROOT and the current working - # different have different mount drives (on Windows). - raise unittest.SkipTest( - f"the current working directory and the Python source code " - f"directory have different mount drives " - f"({cwd_drive} and {root_drive})" - ) - - -skip_if_different_mount_drives() - - -test_tools.skip_if_missing("cases_generator") -with test_tools.imports_under_tool("cases_generator"): - from analyzer import StackItem - from cwriter import CWriter - import parser - from stack import Local, Stack - import tier1_generator - import optimizer_generator - - -def handle_stderr(): - if support.verbose > 1: - return contextlib.nullcontext() - else: - return support.captured_stderr() - - -def parse_src(src): - p = parser.Parser(src, "test.c") - nodes = [] - while node := p.definition(): - nodes.append(node) - return nodes - - -class TestEffects(unittest.TestCase): - def test_effect_sizes(self): - stack = Stack() - inputs = [ - x := StackItem("x", "1"), - y := StackItem("y", "oparg"), - z := StackItem("z", "oparg*2"), - ] - outputs = [ - StackItem("x", "1"), - StackItem("b", "oparg*4"), - StackItem("c", "1"), - ] - null = CWriter.null() - stack.pop(z, null) - stack.pop(y, null) - stack.pop(x, null) - for out in outputs: - stack.push(Local.undefined(out)) - self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") - self.assertEqual(stack.physical_sp.to_c(), "0") - self.assertEqual(stack.logical_sp.to_c(), "1 - oparg - oparg*2 + oparg*4") - - -class TestGeneratedCases(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.maxDiff = None - - self.temp_dir = tempfile.gettempdir() - self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") - self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") - self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") - self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") - self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") - - def tearDown(self) -> None: - for filename in [ - self.temp_input_filename, - self.temp_output_filename, - self.temp_metadata_filename, - self.temp_pymetadata_filename, - self.temp_executor_filename, - ]: - try: - os.remove(filename) - except: - pass - super().tearDown() - - def run_cases_test(self, input: str, expected: str): - with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(parser.BEGIN_MARKER) - temp_input.write(input) - temp_input.write(parser.END_MARKER) - temp_input.flush() - - with handle_stderr(): - tier1_generator.generate_tier1_from_files( - [self.temp_input_filename], self.temp_output_filename, False - ) - - with open(self.temp_output_filename) as temp_output: - lines = temp_output.read() - _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) - instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) - _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) - labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) - actual = instructions.strip() + "\n\n " + labels.strip() - - self.assertEqual(actual.strip(), expected.strip()) - - def test_inst_no_args(self): - input = """ - inst(OP, (--)) { - SPAM(); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - SPAM(); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_inst_one_pop(self): - input = """ - inst(OP, (value --)) { - SPAM(value); - DEAD(value); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef value; - value = stack_pointer[-1]; - SPAM(value); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_inst_one_push(self): - input = """ - inst(OP, (-- res)) { - res = SPAM(); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef res; - res = SPAM(); - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_inst_one_push_one_pop(self): - input = """ - inst(OP, (value -- res)) { - res = SPAM(value); - DEAD(value); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - res = SPAM(value); - stack_pointer[-1] = res; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_binary_op(self): - input = """ - inst(OP, (left, right -- res)) { - res = SPAM(left, right); - INPUTS_DEAD(); - - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - res = SPAM(left, right); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_overlap(self): - input = """ - inst(OP, (left, right -- left, result)) { - result = SPAM(left, right); - INPUTS_DEAD(); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef result; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - result = SPAM(left, right); - stack_pointer[-1] = result; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_predictions(self): - input = """ - inst(OP1, (arg -- res)) { - DEAD(arg); - res = Py_None; - } - inst(OP3, (arg -- res)) { - DEAD(arg); - DEOPT_IF(xxx); - res = Py_None; - } - family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; - """ - output = """ - TARGET(OP1) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP1; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP1); - PREDICTED_OP1:; - _PyStackRef arg; - _PyStackRef res; - arg = stack_pointer[-1]; - res = Py_None; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(OP3) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP3; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP3); - static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); - _PyStackRef arg; - _PyStackRef res; - arg = stack_pointer[-1]; - if (xxx) { - UPDATE_MISS_STATS(OP1); - assert(_PyOpcode_Deopt[opcode] == (OP1)); - JUMP_TO_PREDICTED(OP1); - } - res = Py_None; - stack_pointer[-1] = res; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_sync_sp(self): - input = """ - inst(A, (arg -- res)) { - DEAD(arg); - SYNC_SP(); - escaping_call(); - res = Py_None; - } - inst(B, (arg -- res)) { - DEAD(arg); - res = Py_None; - SYNC_SP(); - escaping_call(); - } - """ - output = """ - TARGET(A) { - #if _Py_TAIL_CALL_INTERP - int opcode = A; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(A); - _PyStackRef arg; - _PyStackRef res; - arg = stack_pointer[-1]; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = Py_None; - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - - TARGET(B) { - #if _Py_TAIL_CALL_INTERP - int opcode = B; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(B); - _PyStackRef arg; - _PyStackRef res; - arg = stack_pointer[-1]; - res = Py_None; - stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); - escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - - def test_pep7_condition(self): - input = """ - inst(OP, (arg1 -- out)) { - if (arg1) - out = 0; - else { - out = 1; - } - } - """ - output = "" - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) - - def test_error_if_plain(self): - input = """ - inst(OP, (--)) { - ERROR_IF(cond); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - if (cond) { - JUMP_TO_LABEL(error); - } - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_error_if_plain_with_comment(self): - input = """ - inst(OP, (--)) { - ERROR_IF(cond); // Comment is ok - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - if (cond) { - JUMP_TO_LABEL(error); - } - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_error_if_pop(self): - input = """ - inst(OP, (left, right -- res)) { - SPAM(left, right); - INPUTS_DEAD(); - ERROR_IF(cond); - res = 0; - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - SPAM(left, right); - if (cond) { - JUMP_TO_LABEL(pop_2_error); - } - res = 0; - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_error_if_pop_with_result(self): - input = """ - inst(OP, (left, right -- res)) { - res = SPAM(left, right); - INPUTS_DEAD(); - ERROR_IF(cond); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - res = SPAM(left, right); - if (cond) { - JUMP_TO_LABEL(pop_2_error); - } - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_cache_effect(self): - input = """ - inst(OP, (counter/1, extra/2, value --)) { - DEAD(value); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(OP); - _PyStackRef value; - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - uint32_t extra = read_u32(&this_instr[2].cache); - (void)extra; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_suppress_dispatch(self): - input = """ - label(somewhere) { - } - - inst(OP, (--)) { - goto somewhere; - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - JUMP_TO_LABEL(somewhere); - } - - LABEL(somewhere) - { - } - """ - self.run_cases_test(input, output) - - def test_macro_instruction(self): - input = """ - inst(OP1, (counter/1, left, right -- left, right)) { - op1(left, right); - } - op(OP2, (extra/2, arg2, left, right -- res)) { - res = op2(arg2, left, right); - INPUTS_DEAD(); - } - macro(OP) = OP1 + cache/2 + OP2; - inst(OP3, (unused/5, arg2, left, right -- res)) { - res = op3(arg2, left, right); - INPUTS_DEAD(); - } - family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(OP); - PREDICTED_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef arg2; - _PyStackRef res; - // _OP1 - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - _PyFrame_SetStackPointer(frame, stack_pointer); - op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 2 cache entries */ - // OP2 - { - arg2 = stack_pointer[-3]; - uint32_t extra = read_u32(&this_instr[4].cache); - (void)extra; - _PyFrame_SetStackPointer(frame, stack_pointer); - res = op2(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-3] = res; - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - - TARGET(OP1) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(OP1); - _PyStackRef left; - _PyStackRef right; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - _PyFrame_SetStackPointer(frame, stack_pointer); - op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - - TARGET(OP3) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP3; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(OP3); - static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); - _PyStackRef arg2; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - /* Skip 5 cache entries */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - arg2 = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - res = op3(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-3] = res; - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_unused_caches(self): - input = """ - inst(OP, (unused/1, unused/2 --)) { - body; - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(OP); - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - body; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_pseudo_instruction_no_flags(self): - input = """ - pseudo(OP, (in -- out1, out2)) = { - OP1, - }; - - inst(OP1, (--)) { - } - """ - output = """ - TARGET(OP1) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP1; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP1); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_pseudo_instruction_with_flags(self): - input = """ - pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { - OP1, - }; - - inst(OP1, (--)) { - } - """ - output = """ - TARGET(OP1) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP1; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP1); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_pseudo_instruction_as_sequence(self): - input = """ - pseudo(OP, (in -- out1, out2)) = [ - OP1, OP2 - ]; - - inst(OP1, (--)) { - } - - inst(OP2, (--)) { - } - """ - output = """ - TARGET(OP1) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP1; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP1); - DISPATCH(); - } - - TARGET(OP2) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP2; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP2); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - - def test_array_input(self): - input = """ - inst(OP, (below, values[oparg*2], above --)) { - SPAM(values, oparg); - DEAD(below); - DEAD(values); - DEAD(above); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef below; - _PyStackRef *values; - _PyStackRef above; - above = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg*2]; - below = stack_pointer[-2 - oparg*2]; - SPAM(values, oparg); - stack_pointer += -2 - oparg*2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_array_output(self): - input = """ - inst(OP, (unused, unused -- below, values[oparg*3], above)) { - SPAM(values, oparg); - below = 0; - above = 0; - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef below; - _PyStackRef *values; - _PyStackRef above; - values = &stack_pointer[-1]; - SPAM(values, oparg); - below = 0; - above = 0; - stack_pointer[-2] = below; - stack_pointer[-1 + oparg*3] = above; - stack_pointer += oparg*3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_array_input_output(self): - input = """ - inst(OP, (values[oparg] -- values[oparg], above)) { - SPAM(values, oparg); - above = 0; - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef *values; - _PyStackRef above; - values = &stack_pointer[-oparg]; - SPAM(values, oparg); - above = 0; - stack_pointer[0] = above; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_array_error_if(self): - input = """ - inst(OP, (extra, values[oparg] --)) { - DEAD(extra); - DEAD(values); - ERROR_IF(oparg == 0); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef extra; - _PyStackRef *values; - values = &stack_pointer[-oparg]; - extra = stack_pointer[-1 - oparg]; - if (oparg == 0) { - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_LABEL(error); - } - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_macro_push_push(self): - input = """ - op(A, (-- val1)) { - val1 = SPAM(); - } - op(B, (-- val2)) { - val2 = SPAM(); - } - macro(M) = A + B; - """ - output = """ - TARGET(M) { - #if _Py_TAIL_CALL_INTERP - int opcode = M; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(M); - _PyStackRef val1; - _PyStackRef val2; - // A - { - val1 = SPAM(); - } - // B - { - val2 = SPAM(); - } - stack_pointer[0] = val1; - stack_pointer[1] = val2; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_override_inst(self): - input = """ - inst(OP, (--)) { - spam; - } - override inst(OP, (--)) { - ham; - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - ham; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_override_op(self): - input = """ - op(OP, (--)) { - spam; - } - macro(M) = OP; - override op(OP, (--)) { - ham; - } - """ - output = """ - TARGET(M) { - #if _Py_TAIL_CALL_INTERP - int opcode = M; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(M); - ham; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_annotated_inst(self): - input = """ - pure inst(OP, (--)) { - ham; - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - ham; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_annotated_op(self): - input = """ - pure op(OP, (--)) { - SPAM(); - } - macro(M) = OP; - """ - output = """ - TARGET(M) { - #if _Py_TAIL_CALL_INTERP - int opcode = M; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(M); - SPAM(); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - input = """ - pure register specializing op(OP, (--)) { - SPAM(); - } - macro(M) = OP; - """ - self.run_cases_test(input, output) - - def test_deopt_and_exit(self): - input = """ - pure op(OP, (arg1 -- out)) { - DEOPT_IF(1); - EXIT_IF(1); - } - """ - output = "" - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) - - def test_array_of_one(self): - input = """ - inst(OP, (arg[1] -- out[1])) { - out[0] = arg[0]; - DEAD(arg); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef *arg; - _PyStackRef *out; - arg = &stack_pointer[-1]; - out = &stack_pointer[-1]; - out[0] = arg[0]; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_unused_cached_value(self): - input = """ - op(FIRST, (arg1 -- out)) { - out = arg1; - } - - op(SECOND, (unused -- unused)) { - } - - macro(BOTH) = FIRST + SECOND; - """ - output = """ - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) - - def test_unused_named_values(self): - input = """ - op(OP, (named -- named)) { - } - - macro(INST) = OP; - """ - output = """ - TARGET(INST) { - #if _Py_TAIL_CALL_INTERP - int opcode = INST; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INST); - DISPATCH(); - } - - """ - self.run_cases_test(input, output) - - def test_used_unused_used(self): - input = """ - op(FIRST, (w -- w)) { - USE(w); - } - - op(SECOND, (x -- x)) { - } - - op(THIRD, (y -- y)) { - USE(y); - } - - macro(TEST) = FIRST + SECOND + THIRD; - """ - output = """ - TARGET(TEST) { - #if _Py_TAIL_CALL_INTERP - int opcode = TEST; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(TEST); - _PyStackRef w; - _PyStackRef y; - // FIRST - { - w = stack_pointer[-1]; - USE(w); - } - // SECOND - { - } - // THIRD - { - y = w; - USE(y); - } - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_unused_used_used(self): - input = """ - op(FIRST, (w -- w)) { - } - - op(SECOND, (x -- x)) { - USE(x); - } - - op(THIRD, (y -- y)) { - USE(y); - } - - macro(TEST) = FIRST + SECOND + THIRD; - """ - output = """ - TARGET(TEST) { - #if _Py_TAIL_CALL_INTERP - int opcode = TEST; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(TEST); - _PyStackRef x; - _PyStackRef y; - // FIRST - { - } - // SECOND - { - x = stack_pointer[-1]; - USE(x); - } - // THIRD - { - y = x; - USE(y); - } - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_flush(self): - input = """ - op(FIRST, ( -- a, b)) { - a = 0; - b = 1; - } - - op(SECOND, (a, b -- )) { - USE(a, b); - INPUTS_DEAD(); - } - - macro(TEST) = FIRST + flush + SECOND; - """ - output = """ - TARGET(TEST) { - #if _Py_TAIL_CALL_INTERP - int opcode = TEST; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(TEST); - _PyStackRef a; - _PyStackRef b; - // FIRST - { - a = 0; - b = 1; - } - // flush - stack_pointer[0] = a; - stack_pointer[1] = b; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - // SECOND - { - USE(a, b); - } - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_pop_on_error_peeks(self): - - input = """ - op(FIRST, (x, y -- a, b)) { - a = x; - DEAD(x); - b = y; - DEAD(y); - } - - op(SECOND, (a, b -- a, b)) { - } - - op(THIRD, (j, k --)) { - INPUTS_DEAD(); // Mark j and k as used - ERROR_IF(cond); - } - - macro(TEST) = FIRST + SECOND + THIRD; - """ - output = """ - TARGET(TEST) { - #if _Py_TAIL_CALL_INTERP - int opcode = TEST; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(TEST); - _PyStackRef x; - _PyStackRef y; - _PyStackRef a; - _PyStackRef b; - // FIRST - { - y = stack_pointer[-1]; - x = stack_pointer[-2]; - a = x; - b = y; - } - // SECOND - { - } - // THIRD - { - if (cond) { - JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_push_then_error(self): - - input = """ - op(FIRST, ( -- a)) { - a = 1; - } - - op(SECOND, (a -- a, b)) { - b = 1; - ERROR_IF(cond); - } - - macro(TEST) = FIRST + SECOND; - """ - - output = """ - TARGET(TEST) { - #if _Py_TAIL_CALL_INTERP - int opcode = TEST; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(TEST); - _PyStackRef a; - _PyStackRef b; - // FIRST - { - a = 1; - } - // SECOND - { - b = 1; - if (cond) { - stack_pointer[0] = a; - stack_pointer[1] = b; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_LABEL(error); - } - } - stack_pointer[0] = a; - stack_pointer[1] = b; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_error_if_true(self): - - input = """ - inst(OP1, ( --)) { - ERROR_IF(true); - } - inst(OP2, ( --)) { - ERROR_IF(1); - } - """ - output = """ - TARGET(OP1) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP1; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP1); - JUMP_TO_LABEL(error); - } - - TARGET(OP2) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP2; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP2); - JUMP_TO_LABEL(error); - } - """ - self.run_cases_test(input, output) - - def test_scalar_array_inconsistency(self): - - input = """ - op(FIRST, ( -- a)) { - a = 1; - } - - op(SECOND, (a[1] -- b)) { - b = 1; - } - - macro(TEST) = FIRST + SECOND; - """ - - output = """ - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) - - def test_array_size_inconsistency(self): - - input = """ - op(FIRST, ( -- a[2])) { - a[0] = 1; - } - - op(SECOND, (a[1] -- b)) { - b = 1; - } - - macro(TEST) = FIRST + SECOND; - """ - - output = """ - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) - - def test_stack_save_reload(self): - - input = """ - inst(BALANCED, ( -- )) { - SAVE_STACK(); - code(); - RELOAD_STACK(); - } - """ - - output = """ - TARGET(BALANCED) { - #if _Py_TAIL_CALL_INTERP - int opcode = BALANCED; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BALANCED); - _PyFrame_SetStackPointer(frame, stack_pointer); - code(); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_stack_save_reload_paired(self): - - input = """ - inst(BALANCED, ( -- )) { - SAVE_STACK(); - RELOAD_STACK(); - } - """ - - output = """ - TARGET(BALANCED) { - #if _Py_TAIL_CALL_INTERP - int opcode = BALANCED; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BALANCED); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_stack_reload_only(self): - - input = """ - inst(BALANCED, ( -- )) { - RELOAD_STACK(); - } - """ - - output = """ - TARGET(BALANCED) { - #if _Py_TAIL_CALL_INTERP - int opcode = BALANCED; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BALANCED); - _PyFrame_SetStackPointer(frame, stack_pointer); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) - - def test_stack_save_only(self): - - input = """ - inst(BALANCED, ( -- )) { - SAVE_STACK(); - } - """ - - output = """ - TARGET(BALANCED) { - #if _Py_TAIL_CALL_INTERP - int opcode = BALANCED; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BALANCED); - _PyFrame_SetStackPointer(frame, stack_pointer); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) - - def test_instruction_size_macro(self): - input = """ - inst(OP, (--)) { - frame->return_offset = INSTRUCTION_SIZE; - } - """ - - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - frame->return_offset = 1u ; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - # Two instructions of different sizes referencing the same - # uop containing the `INSTRUCTION_SIZE` macro is not allowed. - input = """ - inst(OP, (--)) { - frame->return_offset = INSTRUCTION_SIZE; - } - macro(OP2) = unused/1 + OP; - """ - - output = "" # No output needed as this should raise an error. - with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): - self.run_cases_test(input, output) - - def test_escaping_call_next_to_cmacro(self): - input = """ - inst(OP, (--)) { - #ifdef Py_GIL_DISABLED - escaping_call(); - #else - another_escaping_call(); - #endif - yet_another_escaping_call(); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); - #else - _PyFrame_SetStackPointer(frame, stack_pointer); - another_escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - yet_another_escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_pystackref_frompyobject_new_next_to_cmacro(self): - input = """ - inst(OP, (-- out1, out2)) { - PyObject *obj = SPAM(); - #ifdef Py_GIL_DISABLED - out1 = PyStackRef_FromPyObjectNew(obj); - #else - out1 = PyStackRef_FromPyObjectNew(obj); - #endif - out2 = PyStackRef_FromPyObjectNew(obj); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef out1; - _PyStackRef out2; - PyObject *obj = SPAM(); - #ifdef Py_GIL_DISABLED - out1 = PyStackRef_FromPyObjectNew(obj); - #else - out1 = PyStackRef_FromPyObjectNew(obj); - #endif - out2 = PyStackRef_FromPyObjectNew(obj); - stack_pointer[0] = out1; - stack_pointer[1] = out2; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_no_escaping_calls_in_branching_macros(self): - - input = """ - inst(OP, ( -- )) { - DEOPT_IF(escaping_call()); - } - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, "") - - input = """ - inst(OP, ( -- )) { - EXIT_IF(escaping_call()); - } - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, "") - - input = """ - inst(OP, ( -- )) { - ERROR_IF(escaping_call()); - } - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, "") - - def test_kill_in_wrong_order(self): - input = """ - inst(OP, (a, b -- c)) { - c = b; - PyStackRef_CLOSE(a); - PyStackRef_CLOSE(b); - } - """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, "") - - def test_complex_label(self): - input = """ - label(other_label) { - } - - label(other_label2) { - } - - label(my_label) { - // Comment - do_thing(); - if (complex) { - goto other_label; - } - goto other_label2; - } - """ - - output = """ - LABEL(other_label) - { - } - - LABEL(other_label2) - { - } - - LABEL(my_label) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - do_thing(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (complex) { - JUMP_TO_LABEL(other_label); - } - JUMP_TO_LABEL(other_label2); - } - """ - self.run_cases_test(input, output) - - def test_spilled_label(self): - input = """ - spilled label(one) { - RELOAD_STACK(); - goto two; - } - - label(two) { - SAVE_STACK(); - goto one; - } - """ - - output = """ - LABEL(one) - { - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(two); - } - - LABEL(two) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMP_TO_LABEL(one); - } - """ - self.run_cases_test(input, output) - - - def test_incorrect_spills(self): - input1 = """ - spilled label(one) { - goto two; - } - - label(two) { - } - """ - - input2 = """ - spilled label(one) { - } - - label(two) { - goto one; - } - """ - with self.assertRaisesRegex(SyntaxError, ".*reload.*"): - self.run_cases_test(input1, "") - with self.assertRaisesRegex(SyntaxError, ".*spill.*"): - self.run_cases_test(input2, "") - - - def test_multiple_labels(self): - input = """ - label(my_label_1) { - // Comment - do_thing1(); - goto my_label_2; - } - - label(my_label_2) { - // Comment - do_thing2(); - goto my_label_1; - } - """ - - output = """ - LABEL(my_label_1) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - do_thing1(); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(my_label_2); - } - - LABEL(my_label_2) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - do_thing2(); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(my_label_1); - } - """ - self.run_cases_test(input, output) - - def test_reassigning_live_inputs(self): - input = """ - inst(OP, (in -- in)) { - in = 0; - } - """ - - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef in; - in = stack_pointer[-1]; - in = 0; - stack_pointer[-1] = in; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - def test_reassigning_dead_inputs(self): - input = """ - inst(OP, (in -- )) { - temp = use(in); - DEAD(in); - in = temp; - PyStackRef_CLOSE(in); - } - """ - output = """ - TARGET(OP) { - #if _Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef in; - in = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - temp = use(in); - stack_pointer = _PyFrame_GetStackPointer(frame); - in = temp; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(in); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - """ - self.run_cases_test(input, output) - - -class TestGeneratedAbstractCases(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.maxDiff = None - - self.temp_dir = tempfile.gettempdir() - self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") - self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") - self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") - - def tearDown(self) -> None: - for filename in [ - self.temp_input_filename, - self.temp_input2_filename, - self.temp_output_filename, - ]: - try: - os.remove(filename) - except: - pass - super().tearDown() - - def run_cases_test(self, input: str, input2: str, expected: str): - with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(parser.BEGIN_MARKER) - temp_input.write(input) - temp_input.write(parser.END_MARKER) - temp_input.flush() - - with open(self.temp_input2_filename, "w+") as temp_input: - temp_input.write(parser.BEGIN_MARKER) - temp_input.write(input2) - temp_input.write(parser.END_MARKER) - temp_input.flush() - - with handle_stderr(): - optimizer_generator.generate_tier2_abstract_from_files( - [self.temp_input_filename, self.temp_input2_filename], - self.temp_output_filename - ) - - with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual = "".join(lines) - self.assertEqual(actual.strip(), expected.strip()) - - def test_overridden_abstract(self): - input = """ - pure op(OP, (--)) { - SPAM(); - } - """ - input2 = """ - pure op(OP, (--)) { - eggs(); - } - """ - output = """ - case OP: { - eggs(); - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_overridden_abstract_args(self): - input = """ - pure op(OP, (arg1 -- out)) { - out = SPAM(arg1); - } - op(OP2, (arg1 -- out)) { - out = EGGS(arg1); - } - """ - input2 = """ - op(OP, (arg1 -- out)) { - out = EGGS(arg1); - } - """ - output = """ - case OP: { - JitOptRef arg1; - JitOptRef out; - arg1 = stack_pointer[-1]; - out = EGGS(arg1); - stack_pointer[-1] = out; - break; - } - - case OP2: { - JitOptRef out; - out = sym_new_not_null(ctx); - stack_pointer[-1] = out; - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_no_overridden_case(self): - input = """ - pure op(OP, (arg1 -- out)) { - out = SPAM(arg1); - } - - pure op(OP2, (arg1 -- out)) { - } - - """ - input2 = """ - pure op(OP2, (arg1 -- out)) { - out = NULL; - } - """ - output = """ - case OP: { - JitOptRef out; - out = sym_new_not_null(ctx); - stack_pointer[-1] = out; - break; - } - - case OP2: { - JitOptRef out; - out = NULL; - stack_pointer[-1] = out; - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_missing_override_failure(self): - input = """ - pure op(OP, (arg1 -- out)) { - SPAM(); - } - """ - input2 = """ - pure op(OTHER, (arg1 -- out)) { - } - """ - output = """ - """ - with self.assertRaisesRegex(ValueError, "All abstract uops"): - self.run_cases_test(input, input2, output) - - def test_validate_uop_input_length_mismatch(self): - input = """ - op(OP, (arg1 -- out)) { - SPAM(); - } - """ - input2 = """ - op(OP, (arg1, arg2 -- out)) { - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Must have the same number of inputs"): - self.run_cases_test(input, input2, output) - - def test_validate_uop_output_length_mismatch(self): - input = """ - op(OP, (arg1 -- out)) { - SPAM(); - } - """ - input2 = """ - op(OP, (arg1 -- out1, out2)) { - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Must have the same number of outputs"): - self.run_cases_test(input, input2, output) - - def test_validate_uop_input_name_mismatch(self): - input = """ - op(OP, (foo -- out)) { - SPAM(); - } - """ - input2 = """ - op(OP, (bar -- out)) { - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Inputs must have equal names"): - self.run_cases_test(input, input2, output) - - def test_validate_uop_output_name_mismatch(self): - input = """ - op(OP, (arg1 -- foo)) { - SPAM(); - } - """ - input2 = """ - op(OP, (arg1 -- bar)) { - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Outputs must have equal names"): - self.run_cases_test(input, input2, output) - - def test_validate_uop_unused_input(self): - input = """ - op(OP, (unused -- )) { - } - """ - input2 = """ - op(OP, (foo -- )) { - } - """ - output = """ - case OP: { - CHECK_STACK_BOUNDS(-1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - """ - self.run_cases_test(input, input2, output) - - input = """ - op(OP, (foo -- )) { - } - """ - input2 = """ - op(OP, (unused -- )) { - } - """ - output = """ - case OP: { - CHECK_STACK_BOUNDS(-1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_validate_uop_unused_output(self): - input = """ - op(OP, ( -- unused)) { - } - """ - input2 = """ - op(OP, ( -- foo)) { - foo = NULL; - } - """ - output = """ - case OP: { - JitOptRef foo; - foo = NULL; - CHECK_STACK_BOUNDS(1); - stack_pointer[0] = foo; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - """ - self.run_cases_test(input, input2, output) - - input = """ - op(OP, ( -- foo)) { - foo = NULL; - } - """ - input2 = """ - op(OP, ( -- unused)) { - } - """ - output = """ - case OP: { - CHECK_STACK_BOUNDS(1); - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_validate_uop_input_size_mismatch(self): - input = """ - op(OP, (arg1[2] -- )) { - } - """ - input2 = """ - op(OP, (arg1[4] -- )) { - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Inputs must have equal sizes"): - self.run_cases_test(input, input2, output) - - def test_validate_uop_output_size_mismatch(self): - input = """ - op(OP, ( -- out[2])) { - } - """ - input2 = """ - op(OP, ( -- out[4])) { - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Outputs must have equal sizes"): - self.run_cases_test(input, input2, output) - - def test_validate_uop_unused_size_mismatch(self): - input = """ - op(OP, (foo[2] -- )) { - } - """ - input2 = """ - op(OP, (unused[4] -- )) { - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Inputs must have equal sizes"): - self.run_cases_test(input, input2, output) - - def test_pure_uop_body_copied_in(self): - # Note: any non-escaping call works. - # In this case, we use PyStackRef_IsNone. - input = """ - pure op(OP, (foo -- res)) { - res = PyStackRef_IsNone(foo); - } - """ - input2 = """ - op(OP, (foo -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); - res = sym_new_known(ctx, foo); - } - """ - output = """ - case OP: { - JitOptRef foo; - JitOptRef res; - foo = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, foo) - ) { - JitOptRef foo_sym = foo; - _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - res_stackref = PyStackRef_IsNone(foo); - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - stack_pointer[-1] = res; - break; - } - res = sym_new_known(ctx, foo); - stack_pointer[-1] = res; - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_pure_uop_body_copied_in_deopt(self): - # Note: any non-escaping call works. - # In this case, we use PyStackRef_IsNone. - input = """ - pure op(OP, (foo -- res)) { - DEOPT_IF(PyStackRef_IsNull(foo)); - res = foo; - } - """ - input2 = """ - op(OP, (foo -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); - res = foo; - } - """ - output = """ - case OP: { - JitOptRef foo; - JitOptRef res; - foo = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, foo) - ) { - JitOptRef foo_sym = foo; - _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - if (PyStackRef_IsNull(foo)) { - ctx->done = true; - break; - } - res_stackref = foo; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - stack_pointer[-1] = res; - break; - } - res = foo; - stack_pointer[-1] = res; - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_pure_uop_body_copied_in_error_if(self): - # Note: any non-escaping call works. - # In this case, we use PyStackRef_IsNone. - input = """ - pure op(OP, (foo -- res)) { - ERROR_IF(PyStackRef_IsNull(foo)); - res = foo; - } - """ - input2 = """ - op(OP, (foo -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); - res = foo; - } - """ - output = """ - case OP: { - JitOptRef foo; - JitOptRef res; - foo = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, foo) - ) { - JitOptRef foo_sym = foo; - _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - if (PyStackRef_IsNull(foo)) { - goto error; - } - res_stackref = foo; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - stack_pointer[-1] = res; - break; - } - res = foo; - stack_pointer[-1] = res; - break; - } - """ - self.run_cases_test(input, input2, output) - - - def test_replace_opcode_uop_body_copied_in_complex(self): - input = """ - pure op(OP, (foo -- res)) { - if (foo) { - res = PyStackRef_IsNone(foo); - } - else { - res = 1; - } - } - """ - input2 = """ - op(OP, (foo -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); - res = sym_new_known(ctx, foo); - } - """ - output = """ - case OP: { - JitOptRef foo; - JitOptRef res; - foo = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, foo) - ) { - JitOptRef foo_sym = foo; - _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - if (foo) { - res_stackref = PyStackRef_IsNone(foo); - } - else { - res_stackref = 1; - } - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - stack_pointer[-1] = res; - break; - } - res = sym_new_known(ctx, foo); - stack_pointer[-1] = res; - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_replace_opcode_escaping_uop_body_copied_in_complex(self): - input = """ - pure op(OP, (foo -- res)) { - if (foo) { - res = ESCAPING_CODE(foo); - } - else { - res = 1; - } - } - """ - input2 = """ - op(OP, (foo -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); - res = sym_new_known(ctx, foo); - } - """ - output = """ - case OP: { - JitOptRef foo; - JitOptRef res; - foo = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, foo) - ) { - JitOptRef foo_sym = foo; - _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - if (foo) { - res_stackref = ESCAPING_CODE(foo); - } - else { - res_stackref = 1; - } - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - stack_pointer[-1] = res; - break; - } - res = sym_new_known(ctx, foo); - stack_pointer[-1] = res; - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_replace_opcode_binop_one_output(self): - input = """ - pure op(OP, (left, right -- res)) { - res = foo(left, right); - } - """ - input2 = """ - op(OP, (left, right -- res)) { - res = sym_new_non_null(ctx, foo); - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } - """ - output = """ - case OP: { - JitOptRef right; - JitOptRef left; - JitOptRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - res = sym_new_non_null(ctx, foo); - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - res_stackref = foo(left, right); - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_replace_opcode_binop_one_output_insert(self): - input = """ - pure op(OP, (left, right -- res, l, r)) { - res = foo(left, right); - l = left; - r = right; - } - """ - input2 = """ - op(OP, (left, right -- res, l, r)) { - res = sym_new_non_null(ctx, foo); - l = left; - r = right; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); - } - """ - output = """ - case OP: { - JitOptRef right; - JitOptRef left; - JitOptRef res; - JitOptRef l; - JitOptRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - res = sym_new_non_null(ctx, foo); - l = left; - r = right; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - _PyStackRef l_stackref; - _PyStackRef r_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - res_stackref = foo(left, right); - l_stackref = left; - r_stackref = right; - /* End of uop copied from bytecodes for constant evaluation */ - (void)l_stackref; - (void)r_stackref; - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - CHECK_STACK_BOUNDS(1); - stack_pointer[-2] = res; - stack_pointer[-1] = l; - stack_pointer[0] = r; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - CHECK_STACK_BOUNDS(1); - stack_pointer[-2] = res; - stack_pointer[-1] = l; - stack_pointer[0] = r; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_replace_opcode_unaryop_one_output_insert(self): - input = """ - pure op(OP, (left -- res, l)) { - res = foo(left); - l = left; - } - """ - input2 = """ - op(OP, (left -- res, l)) { - res = sym_new_non_null(ctx, foo); - l = left; - REPLACE_OPCODE_IF_EVALUATES_PURE(left, res); - } - """ - output = """ - case OP: { - JitOptRef left; - JitOptRef res; - JitOptRef l; - left = stack_pointer[-1]; - res = sym_new_non_null(ctx, foo); - l = left; - if ( - sym_is_safe_const(ctx, left) - ) { - JitOptRef left_sym = left; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef res_stackref; - _PyStackRef l_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - res_stackref = foo(left); - l_stackref = left; - /* End of uop copied from bytecodes for constant evaluation */ - (void)l_stackref; - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - CHECK_STACK_BOUNDS(1); - stack_pointer[-1] = res; - stack_pointer[0] = l; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - CHECK_STACK_BOUNDS(1); - stack_pointer[-1] = res; - stack_pointer[0] = l; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } - """ - self.run_cases_test(input, input2, output) - - def test_replace_opocode_uop_reject_array_effects(self): - input = """ - pure op(OP, (foo[2] -- res)) { - if (foo) { - res = PyStackRef_IsNone(foo); - } - else { - res = 1; - } - } - """ - input2 = """ - op(OP, (foo[2] -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); - res = sym_new_unknown(ctx); - } - """ - output = """ - """ - with self.assertRaisesRegex(SyntaxError, - "Pure evaluation cannot take array-like inputs"): - self.run_cases_test(input, input2, output) - -if __name__ == "__main__": - unittest.main() +import contextlib +import os +import sys +import tempfile +import unittest + +from test import support +from test import test_tools + + +def skip_if_different_mount_drives(): + if sys.platform != "win32": + return + ROOT = os.path.dirname(os.path.dirname(__file__)) + root_drive = os.path.splitroot(ROOT)[0] + cwd_drive = os.path.splitroot(os.getcwd())[0] + if root_drive != cwd_drive: + # May raise ValueError if ROOT and the current working + # different have different mount drives (on Windows). + raise unittest.SkipTest( + f"the current working directory and the Python source code " + f"directory have different mount drives " + f"({cwd_drive} and {root_drive})" + ) + + +skip_if_different_mount_drives() + + +test_tools.skip_if_missing("cases_generator") +with test_tools.imports_under_tool("cases_generator"): + from analyzer import StackItem, analyze_forest + from cwriter import CWriter + import parser + from stack import Local, Stack + import tier1_generator + import optimizer_generator + + +def handle_stderr(): + if support.verbose > 1: + return contextlib.nullcontext() + else: + return support.captured_stderr() + + +def parse_src(src): + p = parser.Parser(src, "test.c") + nodes = [] + while node := p.definition(): + nodes.append(node) + return nodes + + +class TestEffects(unittest.TestCase): + def test_effect_sizes(self): + stack = Stack() + inputs = [ + x := StackItem("x", "1"), + y := StackItem("y", "oparg"), + z := StackItem("z", "oparg*2"), + ] + outputs = [ + StackItem("x", "1"), + StackItem("b", "oparg*4"), + StackItem("c", "1"), + ] + null = CWriter.null() + stack.pop(z, null) + stack.pop(y, null) + stack.pop(x, null) + for out in outputs: + stack.push(Local.undefined(out)) + self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") + self.assertEqual(stack.physical_sp.to_c(), "0") + self.assertEqual(stack.logical_sp.to_c(), "1 - oparg - oparg*2 + oparg*4") + + +class TestGeneratedCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") + self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") + self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_output_filename, + self.temp_metadata_filename, + self.temp_pymetadata_filename, + self.temp_executor_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + tier1_generator.generate_tier1_from_files( + [self.temp_input_filename], self.temp_output_filename, False + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) + labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) + actual = instructions.strip() + "\n\n " + labels.strip() + + self.assertEqual(actual.strip(), expected.strip()) + + def test_inst_no_args(self): + input = """ + inst(OP, (--)) { + SPAM(); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + SPAM(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_pop(self): + input = """ + inst(OP, (value --)) { + SPAM(value); + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; + value = stack_pointer[-1]; + SPAM(value); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_push(self): + input = """ + inst(OP, (-- res)) { + res = SPAM(); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef res; + res = SPAM(); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_push_one_pop(self): + input = """ + inst(OP, (value -- res)) { + res = SPAM(value); + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + res = SPAM(value); + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_binary_op(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_overlap(self): + input = """ + inst(OP, (left, right -- left, result)) { + result = SPAM(left, right); + INPUTS_DEAD(); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef result; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + result = SPAM(left, right); + stack_pointer[-1] = result; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_predictions(self): + input = """ + inst(OP1, (arg -- res)) { + DEAD(arg); + res = Py_None; + } + inst(OP3, (arg -- res)) { + DEAD(arg); + DEOPT_IF(xxx); + res = Py_None; + } + family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; + """ + output = """ + TARGET(OP1) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + PREDICTED_OP1:; + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(OP3) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP3; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + if (xxx) { + UPDATE_MISS_STATS(OP1); + assert(_PyOpcode_Deopt[opcode] == (OP1)); + JUMP_TO_PREDICTED(OP1); + } + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_sync_sp(self): + input = """ + inst(A, (arg -- res)) { + DEAD(arg); + SYNC_SP(); + escaping_call(); + res = Py_None; + } + inst(B, (arg -- res)) { + DEAD(arg); + res = Py_None; + SYNC_SP(); + escaping_call(); + } + """ + output = """ + TARGET(A) { + #if _Py_TAIL_CALL_INTERP + int opcode = A; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(A); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = Py_None; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(B) { + #if _Py_TAIL_CALL_INTERP + int opcode = B; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(B); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + res = Py_None; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + + def test_pep7_condition(self): + input = """ + inst(OP, (arg1 -- out)) { + if (arg1) + out = 0; + else { + out = 1; + } + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_error_if_plain(self): + input = """ + inst(OP, (--)) { + ERROR_IF(cond); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + if (cond) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_plain_with_comment(self): + input = """ + inst(OP, (--)) { + ERROR_IF(cond); // Comment is ok + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + if (cond) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop(self): + input = """ + inst(OP, (left, right -- res)) { + SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond); + res = 0; + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + SPAM(left, right); + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + res = 0; + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop_with_result(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_cache_effect(self): + input = """ + inst(OP, (counter/1, extra/2, value --)) { + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + _PyStackRef value; + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t extra = read_u32(&this_instr[2].cache); + (void)extra; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_suppress_dispatch(self): + input = """ + label(somewhere) { + } + + inst(OP, (--)) { + goto somewhere; + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + JUMP_TO_LABEL(somewhere); + } + + LABEL(somewhere) + { + } + """ + self.run_cases_test(input, output) + + def test_macro_instruction(self): + input = """ + inst(OP1, (counter/1, left, right -- left, right)) { + op1(left, right); + } + op(OP2, (extra/2, arg2, left, right -- res)) { + res = op2(arg2, left, right); + INPUTS_DEAD(); + } + macro(OP) = OP1 + cache/2 + OP2; + inst(OP3, (unused/5, arg2, left, right -- res)) { + res = op3(arg2, left, right); + INPUTS_DEAD(); + } + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP); + PREDICTED_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef arg2; + _PyStackRef res; + // _OP1 + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 2 cache entries */ + // OP2 + { + arg2 = stack_pointer[-3]; + uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(OP1) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(OP1); + _PyStackRef left; + _PyStackRef right; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(OP3) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP3; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); + _PyStackRef arg2; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 5 cache entries */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + arg2 = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op3(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_caches(self): + input = """ + inst(OP, (unused/1, unused/2 --)) { + body; + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + body; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_no_flags(self): + input = """ + pseudo(OP, (in -- out1, out2)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_with_flags(self): + input = """ + pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_as_sequence(self): + input = """ + pseudo(OP, (in -- out1, out2)) = [ + OP1, OP2 + ]; + + inst(OP1, (--)) { + } + + inst(OP2, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + + TARGET(OP2) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + + def test_array_input(self): + input = """ + inst(OP, (below, values[oparg*2], above --)) { + SPAM(values, oparg); + DEAD(below); + DEAD(values); + DEAD(above); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; + above = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg*2]; + below = stack_pointer[-2 - oparg*2]; + SPAM(values, oparg); + stack_pointer += -2 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_output(self): + input = """ + inst(OP, (unused, unused -- below, values[oparg*3], above)) { + SPAM(values, oparg); + below = 0; + above = 0; + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-1]; + SPAM(values, oparg); + below = 0; + above = 0; + stack_pointer[-2] = below; + stack_pointer[-1 + oparg*3] = above; + stack_pointer += oparg*3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_input_output(self): + input = """ + inst(OP, (values[oparg] -- values[oparg], above)) { + SPAM(values, oparg); + above = 0; + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-oparg]; + SPAM(values, oparg); + above = 0; + stack_pointer[0] = above; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_error_if(self): + input = """ + inst(OP, (extra, values[oparg] --)) { + DEAD(extra); + DEAD(values); + ERROR_IF(oparg == 0); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef extra; + _PyStackRef *values; + values = &stack_pointer[-oparg]; + extra = stack_pointer[-1 - oparg]; + if (oparg == 0) { + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_macro_push_push(self): + input = """ + op(A, (-- val1)) { + val1 = SPAM(); + } + op(B, (-- val2)) { + val2 = SPAM(); + } + macro(M) = A + B; + """ + output = """ + TARGET(M) { + #if _Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + _PyStackRef val1; + _PyStackRef val2; + // A + { + val1 = SPAM(); + } + // B + { + val2 = SPAM(); + } + stack_pointer[0] = val1; + stack_pointer[1] = val2; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_inst(self): + input = """ + inst(OP, (--)) { + spam; + } + override inst(OP, (--)) { + ham; + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_op(self): + input = """ + op(OP, (--)) { + spam; + } + macro(M) = OP; + override op(OP, (--)) { + ham; + } + """ + output = """ + TARGET(M) { + #if _Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_annotated_inst(self): + input = """ + pure inst(OP, (--)) { + ham; + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_annotated_op(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + macro(M) = OP; + """ + output = """ + TARGET(M) { + #if _Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + SPAM(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + input = """ + pure register specializing op(OP, (--)) { + SPAM(); + } + macro(M) = OP; + """ + self.run_cases_test(input, output) + + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_of_one(self): + input = """ + inst(OP, (arg[1] -- out[1])) { + out[0] = arg[0]; + DEAD(arg); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef *out; + arg = &stack_pointer[-1]; + out = &stack_pointer[-1]; + out[0] = arg[0]; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_cached_value(self): + input = """ + op(FIRST, (arg1 -- out)) { + out = arg1; + } + + op(SECOND, (unused -- unused)) { + } + + macro(BOTH) = FIRST + SECOND; + """ + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_unused_named_values(self): + input = """ + op(OP, (named -- named)) { + } + + macro(INST) = OP; + """ + output = """ + TARGET(INST) { + #if _Py_TAIL_CALL_INTERP + int opcode = INST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INST); + DISPATCH(); + } + + """ + self.run_cases_test(input, output) + + def test_used_unused_used(self): + input = """ + op(FIRST, (w -- w)) { + USE(w); + } + + op(SECOND, (x -- x)) { + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef w; + _PyStackRef y; + // FIRST + { + w = stack_pointer[-1]; + USE(w); + } + // SECOND + { + } + // THIRD + { + y = w; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_used_used(self): + input = """ + op(FIRST, (w -- w)) { + } + + op(SECOND, (x -- x)) { + USE(x); + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + // FIRST + { + } + // SECOND + { + x = stack_pointer[-1]; + USE(x); + } + // THIRD + { + y = x; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_flush(self): + input = """ + op(FIRST, ( -- a, b)) { + a = 0; + b = 1; + } + + op(SECOND, (a, b -- )) { + USE(a, b); + INPUTS_DEAD(); + } + + macro(TEST) = FIRST + flush + SECOND; + """ + output = """ + TARGET(TEST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 0; + b = 1; + } + // flush + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + // SECOND + { + USE(a, b); + } + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_on_error_peeks(self): + + input = """ + op(FIRST, (x, y -- a, b)) { + a = x; + DEAD(x); + b = y; + DEAD(y); + } + + op(SECOND, (a, b -- a, b)) { + } + + op(THIRD, (j, k --)) { + INPUTS_DEAD(); // Mark j and k as used + ERROR_IF(cond); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + _PyStackRef a; + _PyStackRef b; + // FIRST + { + y = stack_pointer[-1]; + x = stack_pointer[-2]; + a = x; + b = y; + } + // SECOND + { + } + // THIRD + { + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_push_then_error(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a -- a, b)) { + b = 1; + ERROR_IF(cond); + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + TARGET(TEST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 1; + } + // SECOND + { + b = 1; + if (cond) { + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_true(self): + + input = """ + inst(OP1, ( --)) { + ERROR_IF(true); + } + inst(OP2, ( --)) { + ERROR_IF(1); + } + """ + output = """ + TARGET(OP1) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + JUMP_TO_LABEL(error); + } + + TARGET(OP2) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + JUMP_TO_LABEL(error); + } + """ + self.run_cases_test(input, output) + + def test_scalar_array_inconsistency(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_size_inconsistency(self): + + input = """ + op(FIRST, ( -- a[2])) { + a[0] = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_reload(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + code(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if _Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + code(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_save_reload_paired(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if _Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_reload_only(self): + + input = """ + inst(BALANCED, ( -- )) { + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if _Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_only(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if _Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_instruction_size_macro(self): + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + """ + + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + frame->return_offset = 1u ; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + # Two instructions of different sizes referencing the same + # uop containing the `INSTRUCTION_SIZE` macro is not allowed. + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + macro(OP2) = unused/1 + OP; + """ + + output = "" # No output needed as this should raise an error. + with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): + self.run_cases_test(input, output) + + def test_escaping_call_next_to_cmacro(self): + input = """ + inst(OP, (--)) { + #ifdef Py_GIL_DISABLED + escaping_call(); + #else + another_escaping_call(); + #endif + yet_another_escaping_call(); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #else + _PyFrame_SetStackPointer(frame, stack_pointer); + another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + yet_another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pystackref_frompyobject_new_next_to_cmacro(self): + input = """ + inst(OP, (-- out1, out2)) { + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef out1; + _PyStackRef out2; + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = out1; + stack_pointer[1] = out2; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_no_escaping_calls_in_branching_macros(self): + + input = """ + inst(OP, ( -- )) { + DEOPT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + EXIT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + ERROR_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_kill_in_wrong_order(self): + input = """ + inst(OP, (a, b -- c)) { + c = b; + PyStackRef_CLOSE(a); + PyStackRef_CLOSE(b); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_complex_label(self): + input = """ + label(other_label) { + } + + label(other_label2) { + } + + label(my_label) { + // Comment + do_thing(); + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + + output = """ + LABEL(other_label) + { + } + + LABEL(other_label2) + { + } + + LABEL(my_label) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (complex) { + JUMP_TO_LABEL(other_label); + } + JUMP_TO_LABEL(other_label2); + } + """ + self.run_cases_test(input, output) + + def test_spilled_label(self): + input = """ + spilled label(one) { + RELOAD_STACK(); + goto two; + } + + label(two) { + SAVE_STACK(); + goto one; + } + """ + + output = """ + LABEL(one) + { + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(two); + } + + LABEL(two) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMP_TO_LABEL(one); + } + """ + self.run_cases_test(input, output) + + + def test_incorrect_spills(self): + input1 = """ + spilled label(one) { + goto two; + } + + label(two) { + } + """ + + input2 = """ + spilled label(one) { + } + + label(two) { + goto one; + } + """ + with self.assertRaisesRegex(SyntaxError, ".*reload.*"): + self.run_cases_test(input1, "") + with self.assertRaisesRegex(SyntaxError, ".*spill.*"): + self.run_cases_test(input2, "") + + + def test_multiple_labels(self): + input = """ + label(my_label_1) { + // Comment + do_thing1(); + goto my_label_2; + } + + label(my_label_2) { + // Comment + do_thing2(); + goto my_label_1; + } + """ + + output = """ + LABEL(my_label_1) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing1(); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_2); + } + + LABEL(my_label_2) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing2(); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_1); + } + """ + self.run_cases_test(input, output) + + def test_reassigning_live_inputs(self): + input = """ + inst(OP, (in -- in)) { + in = 0; + } + """ + + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef in; + in = stack_pointer[-1]; + in = 0; + stack_pointer[-1] = in; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_reassigning_dead_inputs(self): + input = """ + inst(OP, (in -- )) { + temp = use(in); + DEAD(in); + in = temp; + PyStackRef_CLOSE(in); + } + """ + output = """ + TARGET(OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef in; + in = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + temp = use(in); + stack_pointer = _PyFrame_GetStackPointer(frame); + in = temp; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(in); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + +class TestGeneratedAbstractCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_input2_filename, + self.temp_output_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, input2: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with open(self.temp_input2_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input2) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + optimizer_generator.generate_tier2_abstract_from_files( + [self.temp_input_filename, self.temp_input2_filename], + self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + self.assertEqual(actual.strip(), expected.strip()) + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + op(OP2, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + output = """ + case OP: { + JitOptRef arg1; + JitOptRef out; + arg1 = stack_pointer[-1]; + out = EGGS(arg1); + stack_pointer[-1] = out; + break; + } + + case OP2: { + JitOptRef out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_no_overridden_case(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + + pure op(OP2, (arg1 -- out)) { + } + + """ + input2 = """ + pure op(OP2, (arg1 -- out)) { + out = NULL; + } + """ + output = """ + case OP: { + JitOptRef out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + + case OP2: { + JitOptRef out; + out = NULL; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_missing_override_failure(self): + input = """ + pure op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + pure op(OTHER, (arg1 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(ValueError, "All abstract uops"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_input_length_mismatch(self): + input = """ + op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1, arg2 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Must have the same number of inputs"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_length_mismatch(self): + input = """ + op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1 -- out1, out2)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Must have the same number of outputs"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_input_name_mismatch(self): + input = """ + op(OP, (foo -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (bar -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal names"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_name_mismatch(self): + input = """ + op(OP, (arg1 -- foo)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1 -- bar)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Outputs must have equal names"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_input(self): + input = """ + op(OP, (unused -- )) { + } + """ + input2 = """ + op(OP, (foo -- )) { + } + """ + output = """ + case OP: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + input = """ + op(OP, (foo -- )) { + } + """ + input2 = """ + op(OP, (unused -- )) { + } + """ + output = """ + case OP: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_output(self): + input = """ + op(OP, ( -- unused)) { + } + """ + input2 = """ + op(OP, ( -- foo)) { + foo = NULL; + } + """ + output = """ + case OP: { + JitOptRef foo; + foo = NULL; + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = foo; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + input = """ + op(OP, ( -- foo)) { + foo = NULL; + } + """ + input2 = """ + op(OP, ( -- unused)) { + } + """ + output = """ + case OP: { + CHECK_STACK_BOUNDS(1); + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_validate_uop_input_size_mismatch(self): + input = """ + op(OP, (arg1[2] -- )) { + } + """ + input2 = """ + op(OP, (arg1[4] -- )) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_size_mismatch(self): + input = """ + op(OP, ( -- out[2])) { + } + """ + input2 = """ + op(OP, ( -- out[4])) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Outputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_size_mismatch(self): + input = """ + op(OP, (foo[2] -- )) { + } + """ + input2 = """ + op(OP, (unused[4] -- )) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + res = PyStackRef_IsNone(foo); + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + res_stackref = PyStackRef_IsNone(foo); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in_deopt(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + DEOPT_IF(PyStackRef_IsNull(foo)); + res = foo; + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); + res = foo; + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (PyStackRef_IsNull(foo)) { + ctx->done = true; + break; + } + res_stackref = foo; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = foo; + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in_error_if(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + ERROR_IF(PyStackRef_IsNull(foo)); + res = foo; + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); + res = foo; + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (PyStackRef_IsNull(foo)) { + goto error; + } + res_stackref = foo; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = foo; + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + + def test_replace_opcode_uop_body_copied_in_complex(self): + input = """ + pure op(OP, (foo -- res)) { + if (foo) { + res = PyStackRef_IsNone(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (foo) { + res_stackref = PyStackRef_IsNone(foo); + } + else { + res_stackref = 1; + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opcode_escaping_uop_body_copied_in_complex(self): + input = """ + pure op(OP, (foo -- res)) { + if (foo) { + res = ESCAPING_CODE(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (foo) { + res_stackref = ESCAPING_CODE(foo); + } + else { + res_stackref = 1; + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opcode_binop_one_output(self): + input = """ + pure op(OP, (left, right -- res)) { + res = foo(left, right); + } + """ + input2 = """ + op(OP, (left, right -- res)) { + res = sym_new_non_null(ctx, foo); + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } + """ + output = """ + case OP: { + JitOptRef right; + JitOptRef left; + JitOptRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = sym_new_non_null(ctx, foo); + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + res_stackref = foo(left, right); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opcode_binop_one_output_insert(self): + input = """ + pure op(OP, (left, right -- res, l, r)) { + res = foo(left, right); + l = left; + r = right; + } + """ + input2 = """ + op(OP, (left, right -- res, l, r)) { + res = sym_new_non_null(ctx, foo); + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res); + } + """ + output = """ + case OP: { + JitOptRef right; + JitOptRef left; + JitOptRef res; + JitOptRef l; + JitOptRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = sym_new_non_null(ctx, foo); + l = left; + r = right; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + _PyStackRef l_stackref; + _PyStackRef r_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + res_stackref = foo(left, right); + l_stackref = left; + r_stackref = right; + /* End of uop copied from bytecodes for constant evaluation */ + (void)l_stackref; + (void)r_stackref; + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opcode_unaryop_one_output_insert(self): + input = """ + pure op(OP, (left -- res, l)) { + res = foo(left); + l = left; + } + """ + input2 = """ + op(OP, (left -- res, l)) { + res = sym_new_non_null(ctx, foo); + l = left; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, res); + } + """ + output = """ + case OP: { + JitOptRef left; + JitOptRef res; + JitOptRef l; + left = stack_pointer[-1]; + res = sym_new_non_null(ctx, foo); + l = left; + if ( + sym_is_safe_const(ctx, left) + ) { + JitOptRef left_sym = left; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef res_stackref; + _PyStackRef l_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + res_stackref = foo(left); + l_stackref = left; + /* End of uop copied from bytecodes for constant evaluation */ + (void)l_stackref; + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + CHECK_STACK_BOUNDS(1); + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + CHECK_STACK_BOUNDS(1); + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opocode_uop_reject_array_effects(self): + input = """ + pure op(OP, (foo[2] -- res)) { + if (foo) { + res = PyStackRef_IsNone(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo[2] -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo, res); + res = sym_new_unknown(ctx); + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Pure evaluation cannot take array-like inputs"): + self.run_cases_test(input, input2, output) + +class TestAnalyzer(unittest.TestCase): + """Tests for analyzer.add_macro() recording-uop placement rules (gh-148285).""" + + def _parse_and_analyze(self, src: str) -> None: + """Parse a raw DSL fragment and run analyze_forest() on it.""" + nodes = parse_src(src) + analyze_forest(nodes) + + # ---- shared DSL fragments ----------------------------------------------- + + _SPECIALIZE_OP = """\ +specializing op(_SPECIALIZE_DUMMY, (counter/1, value -- value)) { +} +""" + _RECORD_OP = """\ +op(_RECORD_DUMMY, (value -- value)) { + RECORD_VALUE(PyStackRef_AsPyObjectBorrow(value)); +} +""" + _WORKER_OP = """\ +op(_WORKER_DUMMY, (value -- res)) { + res = value; +} +""" + + # ---- test cases --------------------------------------------------------- + + def test_recording_uop_after_specializing(self): + """Valid: recording uop directly follows a specializing uop.""" + src = ( + self._SPECIALIZE_OP + + self._RECORD_OP + + self._WORKER_OP + + "macro(VALID_DIRECT) = _SPECIALIZE_DUMMY + _RECORD_DUMMY + _WORKER_DUMMY;\n" + ) + # Must not raise. + self._parse_and_analyze(src) + + def test_recording_uop_after_specializing_with_cache(self): + """Valid: recording uop follows a specializing uop with a cache effect between them. + + CacheEffect entries are transparent — they must not close the gate that + allows a recording uop to follow a specializing uop. + """ + src = ( + self._SPECIALIZE_OP + + self._RECORD_OP + + self._WORKER_OP + + "macro(VALID_CACHE) = _SPECIALIZE_DUMMY + unused/1 + _RECORD_DUMMY + _WORKER_DUMMY;\n" + ) + # Must not raise. + self._parse_and_analyze(src) + + def test_recording_uop_after_non_specializing_raises(self): + """Invalid: recording uop after a plain (non-specializing) uop must be rejected.""" + src = ( + self._SPECIALIZE_OP + + self._RECORD_OP + + self._WORKER_OP + + "macro(INVALID) = _WORKER_DUMMY + _RECORD_DUMMY;\n" + ) + with self.assertRaisesRegex( + SyntaxError, + r"Recording uop _RECORD_DUMMY must be first in macro " + r"or immediately follow a specializing uop", + ): + self._parse_and_analyze(src) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 8b9bc3737260d9..27854c362c33ec 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,1455 +1,1455 @@ -from dataclasses import dataclass -import itertools -import lexer -import parser -import re -from typing import Optional, Callable, Iterator - -from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, MacroIfStmt - -MAX_CACHED_REGISTER = 3 - -@dataclass -class EscapingCall: - stmt: SimpleStmt - call: lexer.Token - kills: lexer.Token | None - -@dataclass -class Properties: - escaping_calls: dict[SimpleStmt, EscapingCall] - escapes: bool - error_with_pop: bool - error_without_pop: bool - deopts: bool - deopts_periodic: bool - oparg: bool - jumps: bool - eval_breaker: bool - needs_this: bool - always_exits: bool - sync_sp: bool - uses_co_consts: bool - uses_co_names: bool - uses_locals: bool - has_free: bool - side_exit: bool - side_exit_at_end: bool - pure: bool - uses_opcode: bool - needs_guard_ip: bool - unpredictable_jump: bool - records_value: bool - tier: int | None = None - const_oparg: int = -1 - needs_prev: bool = False - no_save_ip: bool = False - - def dump(self, indent: str) -> None: - simple_properties = self.__dict__.copy() - del simple_properties["escaping_calls"] - text = "escaping_calls:\n" - for tkns in self.escaping_calls.values(): - text += f"{indent} {tkns}\n" - text += ", ".join([f"{key}: {value}" for (key, value) in simple_properties.items()]) - print(indent, text, sep="") - - @staticmethod - def from_list(properties: list["Properties"]) -> "Properties": - escaping_calls: dict[SimpleStmt, EscapingCall] = {} - for p in properties: - escaping_calls.update(p.escaping_calls) - return Properties( - escaping_calls=escaping_calls, - escapes = any(p.escapes for p in properties), - error_with_pop=any(p.error_with_pop for p in properties), - error_without_pop=any(p.error_without_pop for p in properties), - deopts=any(p.deopts for p in properties), - deopts_periodic=any(p.deopts_periodic for p in properties), - oparg=any(p.oparg for p in properties), - jumps=any(p.jumps for p in properties), - eval_breaker=any(p.eval_breaker for p in properties), - needs_this=any(p.needs_this for p in properties), - always_exits=any(p.always_exits for p in properties), - sync_sp=any(p.sync_sp for p in properties), - uses_co_consts=any(p.uses_co_consts for p in properties), - uses_co_names=any(p.uses_co_names for p in properties), - uses_locals=any(p.uses_locals for p in properties), - uses_opcode=any(p.uses_opcode for p in properties), - has_free=any(p.has_free for p in properties), - side_exit=any(p.side_exit for p in properties), - side_exit_at_end=any(p.side_exit_at_end for p in properties), - pure=all(p.pure for p in properties), - needs_prev=any(p.needs_prev for p in properties), - no_save_ip=all(p.no_save_ip for p in properties), - needs_guard_ip=any(p.needs_guard_ip for p in properties), - unpredictable_jump=any(p.unpredictable_jump for p in properties), - records_value=any(p.records_value for p in properties), - ) - - @property - def infallible(self) -> bool: - return not self.error_with_pop and not self.error_without_pop - -SKIP_PROPERTIES = Properties( - escaping_calls={}, - escapes=False, - error_with_pop=False, - error_without_pop=False, - deopts=False, - deopts_periodic=False, - oparg=False, - jumps=False, - eval_breaker=False, - needs_this=False, - always_exits=False, - sync_sp=False, - uses_co_consts=False, - uses_co_names=False, - uses_locals=False, - uses_opcode=False, - has_free=False, - side_exit=False, - side_exit_at_end=False, - pure=True, - no_save_ip=False, - needs_guard_ip=False, - unpredictable_jump=False, - records_value=False, -) - - -@dataclass -class Skip: - "Unused cache entry" - size: int - - @property - def name(self) -> str: - return f"unused/{self.size}" - - @property - def properties(self) -> Properties: - return SKIP_PROPERTIES - - -class Flush: - @property - def properties(self) -> Properties: - return SKIP_PROPERTIES - - @property - def name(self) -> str: - return "flush" - - @property - def size(self) -> int: - return 0 - - - - -@dataclass -class StackItem: - name: str - size: str - peek: bool = False - used: bool = False - - def __str__(self) -> str: - size = f"[{self.size}]" if self.size else "" - return f"{self.name}{size} {self.peek}" - - def is_array(self) -> bool: - return self.size != "" - - def get_size(self) -> str: - return self.size if self.size else "1" - - -@dataclass -class StackEffect: - inputs: list[StackItem] - outputs: list[StackItem] - - def __str__(self) -> str: - return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" - - -@dataclass -class CacheEntry: - name: str - size: int - - def __str__(self) -> str: - return f"{self.name}/{self.size}" - - -@dataclass -class Uop: - name: str - context: parser.Context | None - annotations: list[str] - stack: StackEffect - caches: list[CacheEntry] - local_stores: list[lexer.Token] - body: BlockStmt - properties: Properties - _size: int = -1 - implicitly_created: bool = False - replicated = range(0) - replicates: "Uop | None" = None - # Size of the instruction(s), only set for uops containing the INSTRUCTION_SIZE macro - instruction_size: int | None = None - - def dump(self, indent: str) -> None: - print( - indent, self.name, ", ".join(self.annotations) if self.annotations else "" - ) - print(indent, self.stack, ", ".join([str(c) for c in self.caches])) - self.properties.dump(" " + indent) - - @property - def size(self) -> int: - if self._size < 0: - self._size = sum(c.size for c in self.caches) - return self._size - - def why_not_viable(self) -> str | None: - if self.name == "_SAVE_RETURN_OFFSET": - return None # Adjusts next_instr, but only in tier 1 code - if "INSTRUMENTED" in self.name: - return "is instrumented" - if "replaced" in self.annotations: - return "is replaced" - if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): - return "has tier 1 control flow" - if self.properties.needs_this: - return "uses the 'this_instr' variable" - if len([c for c in self.caches if c.name != "unused"]) > 2: - return "has too many cache entries" - if self.properties.error_with_pop and self.properties.error_without_pop: - return "has both popping and not-popping errors" - return None - - def is_viable(self) -> bool: - return self.why_not_viable() is None - - def is_super(self) -> bool: - for tkn in self.body.tokens(): - if tkn.kind == "IDENTIFIER" and tkn.text == "oparg1": - return True - return False - - -class Label: - - def __init__(self, name: str, spilled: bool, body: BlockStmt, properties: Properties): - self.name = name - self.spilled = spilled - self.body = body - self.properties = properties - - size:int = 0 - local_stores: list[lexer.Token] = [] - instruction_size = None - - def __str__(self) -> str: - return f"label({self.name})" - - -Part = Uop | Skip | Flush -CodeSection = Uop | Label - - -@dataclass -class Instruction: - where: lexer.Token - name: str - parts: list[Part] - _properties: Properties | None - is_target: bool = False - family: Optional["Family"] = None - opcode: int = -1 - - @property - def properties(self) -> Properties: - if self._properties is None: - self._properties = self._compute_properties() - return self._properties - - def _compute_properties(self) -> Properties: - return Properties.from_list([part.properties for part in self.parts]) - - def dump(self, indent: str) -> None: - print(indent, self.name, "=", ", ".join([part.name for part in self.parts])) - self.properties.dump(" " + indent) - - @property - def size(self) -> int: - return 1 + sum(part.size for part in self.parts) - - def is_super(self) -> bool: - if len(self.parts) != 1: - return False - uop = self.parts[0] - if isinstance(uop, Uop): - return uop.is_super() - else: - return False - - -@dataclass -class PseudoInstruction: - name: str - stack: StackEffect - targets: list[Instruction] - as_sequence: bool - flags: list[str] - opcode: int = -1 - - def dump(self, indent: str) -> None: - print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) - - @property - def properties(self) -> Properties: - return Properties.from_list([i.properties for i in self.targets]) - - -@dataclass -class Family: - name: str - size: str - members: list[Instruction] - - def dump(self, indent: str) -> None: - print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) - - -@dataclass -class Analysis: - instructions: dict[str, Instruction] - uops: dict[str, Uop] - families: dict[str, Family] - pseudos: dict[str, PseudoInstruction] - labels: dict[str, Label] - opmap: dict[str, int] - have_arg: int - min_instrumented: int - - -def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: - # To do -- support file and line output - # Construct a SyntaxError instance from message and token - return lexer.make_syntax_error(message, tkn.filename, tkn.line, tkn.column, "") - - -def override_error( - name: str, - context: parser.Context | None, - prev_context: parser.Context | None, - token: lexer.Token, -) -> SyntaxError: - return analysis_error( - f"Duplicate definition of '{name}' @ {context} " - f"previous definition @ {prev_context}", - token, - ) - - -def convert_stack_item( - item: parser.StackEffect, replace_op_arg_1: str | None -) -> StackItem: - return StackItem(item.name, item.size) - -def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]) -> None: - "Unused items cannot be on the stack above used, non-peek items" - seen_unused = False - for item in reversed(stack): - if item.name == "unused": - seen_unused = True - elif item.peek: - break - elif seen_unused: - raise analysis_error(f"Cannot have used input '{item.name}' below an unused value on the stack", input_names[item.name]) - - -def analyze_stack( - op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None -) -> StackEffect: - inputs: list[StackItem] = [ - convert_stack_item(i, replace_op_arg_1) - for i in op.inputs - if isinstance(i, parser.StackEffect) - ] - outputs: list[StackItem] = [ - convert_stack_item(i, replace_op_arg_1) for i in op.outputs - ] - # Mark variables with matching names at the base of the stack as "peek" - modified = False - input_names: dict[str, lexer.Token] = { i.name : i.first_token for i in op.inputs if i.name != "unused" } - for input, output in itertools.zip_longest(inputs, outputs): - if output is None: - pass - elif input is None: - if output.name in input_names: - raise analysis_error( - f"Reuse of variable '{output.name}' at different stack location", - input_names[output.name]) - elif input.name == output.name: - if not modified: - input.peek = output.peek = True - else: - modified = True - if output.name in input_names: - raise analysis_error( - f"Reuse of variable '{output.name}' at different stack location", - input_names[output.name]) - if isinstance(op, parser.InstDef): - output_names = [out.name for out in outputs] - for input in inputs: - if ( - variable_used(op, input.name) - or variable_used(op, "DECREF_INPUTS") - or (not input.peek and input.name in output_names) - ): - input.used = True - for output in outputs: - if variable_used(op, output.name): - output.used = True - check_unused(inputs, input_names) - return StackEffect(inputs, outputs) - - -def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: - caches: list[parser.CacheEffect] = [ - i for i in inputs if isinstance(i, parser.CacheEffect) - ] - if caches: - # Middle entries are allowed to be unused. Check first and last caches. - for index in (0, -1): - cache = caches[index] - if cache.name == "unused": - position = "First" if index == 0 else "Last" - msg = f"{position} cache entry in op is unused. Move to enclosing macro." - raise analysis_error(msg, cache.tokens[0]) - return [CacheEntry(i.name, int(i.size)) for i in caches] - - -def find_variable_stores(node: parser.InstDef) -> list[lexer.Token]: - res: list[lexer.Token] = [] - outnames = { out.name for out in node.outputs } - innames = { out.name for out in node.inputs } - - def find_stores_in_tokens(tokens: list[lexer.Token], callback: Callable[[lexer.Token], None]) -> None: - while tokens and tokens[0].kind == "COMMENT": - tokens = tokens[1:] - if len(tokens) < 4: - return - if tokens[1].kind == "EQUALS": - if tokens[0].kind == "IDENTIFIER": - name = tokens[0].text - if name in outnames or name in innames: - callback(tokens[0]) - #Passing the address of a local is also a definition - for idx, tkn in enumerate(tokens): - if tkn.kind == "AND": - name_tkn = tokens[idx+1] - if name_tkn.text in outnames: - callback(name_tkn) - - def visit(stmt: Stmt) -> None: - if isinstance(stmt, IfStmt): - def error(tkn: lexer.Token) -> None: - raise analysis_error("Cannot define variable in 'if' condition", tkn) - find_stores_in_tokens(stmt.condition, error) - elif isinstance(stmt, SimpleStmt): - find_stores_in_tokens(stmt.contents, res.append) - - node.block.accept(visit) - return res - - -#def analyze_deferred_refs(node: parser.InstDef) -> dict[lexer.Token, str | None]: - #"""Look for PyStackRef_FromPyObjectNew() calls""" - - #def in_frame_push(idx: int) -> bool: - #for tkn in reversed(node.block.tokens[: idx - 1]): - #if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: - #return False - #if tkn.kind == "IDENTIFIER" and tkn.text == "_PyFrame_PushUnchecked": - #return True - #return False - - #refs: dict[lexer.Token, str | None] = {} - #for idx, tkn in enumerate(node.block.tokens): - #if tkn.kind != "IDENTIFIER" or tkn.text != "PyStackRef_FromPyObjectNew": - #continue - - #if idx == 0 or node.block.tokens[idx - 1].kind != "EQUALS": - #if in_frame_push(idx): - ## PyStackRef_FromPyObjectNew() is called in _PyFrame_PushUnchecked() - #refs[tkn] = None - #continue - #raise analysis_error("Expected '=' before PyStackRef_FromPyObjectNew", tkn) - - #lhs = find_assignment_target(node, idx - 1) - #if len(lhs) == 0: - #raise analysis_error( - #"PyStackRef_FromPyObjectNew() must be assigned to an output", tkn - #) - - #if lhs[0].kind == "TIMES" or any( - #t.kind == "ARROW" or t.kind == "LBRACKET" for t in lhs[1:] - #): - ## Don't handle: *ptr = ..., ptr->field = ..., or ptr[field] = ... - ## Assume that they are visible to the GC. - #refs[tkn] = None - #continue - - #if len(lhs) != 1 or lhs[0].kind != "IDENTIFIER": - #raise analysis_error( - #"PyStackRef_FromPyObjectNew() must be assigned to an output", tkn - #) - - #name = lhs[0].text - #match = ( - #any(var.name == name for var in node.inputs) - #or any(var.name == name for var in node.outputs) - #) - #if not match: - #raise analysis_error( - #f"PyStackRef_FromPyObjectNew() must be assigned to an input or output, not '{name}'", - #tkn, - #) - - #refs[tkn] = name - - #return refs - - -def variable_used(node: parser.CodeDef, name: str) -> bool: - """Determine whether a variable with a given name is used in a node.""" - return any( - token.kind == "IDENTIFIER" and token.text == name for token in node.block.tokens() - ) - - -def oparg_used(node: parser.CodeDef) -> bool: - """Determine whether `oparg` is used in a node.""" - return any( - token.kind == "IDENTIFIER" and token.text == "oparg" for token in node.tokens - ) - - -def tier_variable(node: parser.CodeDef) -> int | None: - """Determine whether a tier variable is used in a node.""" - if isinstance(node, parser.LabelDef): - return None - for token in node.tokens: - if token.kind == "ANNOTATION": - if token.text == "specializing": - return 1 - if re.fullmatch(r"tier\d", token.text): - return int(token.text[-1]) - return None - - -def has_error_with_pop(op: parser.CodeDef) -> bool: - return ( - variable_used(op, "ERROR_IF") - or variable_used(op, "exception_unwind") - ) - - -def has_error_without_pop(op: parser.CodeDef) -> bool: - return ( - variable_used(op, "ERROR_NO_POP") - or variable_used(op, "exception_unwind") - ) - - -NON_ESCAPING_FUNCTIONS = ( - "PyCFunction_GET_FLAGS", - "PyCFunction_GET_FUNCTION", - "PyCFunction_GET_SELF", - "PyCell_GetRef", - "PyCell_New", - "PyCell_SwapTakeRef", - "PyExceptionInstance_Class", - "PyException_GetCause", - "PyException_GetContext", - "PyException_GetTraceback", - "PyFloat_AS_DOUBLE", - "PyFloat_FromDouble", - "PyFunction_GET_CODE", - "PyFunction_GET_GLOBALS", - "PyList_GET_ITEM", - "PyList_GET_SIZE", - "PyList_SET_ITEM", - "PyLong_AsLong", - "PyLong_FromLong", - "PyLong_FromSsize_t", - "PySlice_New", - "PyStackRef_AsPyObjectBorrow", - "PyStackRef_AsPyObjectNew", - "PyStackRef_FromPyObjectNewMortal", - "PyStackRef_AsPyObjectSteal", - "PyStackRef_Borrow", - "PyStackRef_CLEAR", - "PyStackRef_CLOSE_SPECIALIZED", - "PyStackRef_DUP", - "PyStackRef_False", - "PyStackRef_FromPyObjectBorrow", - "PyStackRef_FromPyObjectNew", - "PyStackRef_FromPyObjectSteal", - "PyStackRef_IsExactly", - "PyStackRef_FromPyObjectStealMortal", - "PyStackRef_IsNone", - "PyStackRef_Is", - "PyStackRef_IsHeapSafe", - "PyStackRef_IsTrue", - "PyStackRef_IsFalse", - "PyStackRef_IsNull", - "PyStackRef_MakeHeapSafe", - "PyStackRef_None", - "PyStackRef_RefcountOnObject", - "PyStackRef_TYPE", - "PyStackRef_True", - "PyTuple_GET_ITEM", - "PyTuple_GET_SIZE", - "PyType_HasFeature", - "PyUnicode_Concat", - "PyUnicode_GET_LENGTH", - "PyUnicode_READ_CHAR", - "PyUnicode_IS_COMPACT_ASCII", - "PyUnicode_1BYTE_DATA", - "Py_ARRAY_LENGTH", - "Py_FatalError", - "Py_INCREF", - "Py_IS_TYPE", - "Py_NewRef", - "Py_REFCNT", - "Py_SIZE", - "Py_TYPE", - "Py_UNREACHABLE", - "Py_Unicode_GET_LENGTH", - "_PyCode_CODE", - "_PyDictValues_AddToInsertionOrder", - "_PyErr_Occurred", - "_PyFrame_GetBytecode", - "_PyFrame_GetCode", - "_PyFrame_IsIncomplete", - "_PyFrame_PushUnchecked", - "_PyFrame_SetStackPointer", - "_PyFrame_StackPush", - "_PyFunction_SetVersion", - "_PyGen_GetGeneratorFromFrame", - "gen_try_set_executing", - "_PyInterpreterState_GET", - "_PyList_AppendTakeRef", - "_PyList_ITEMS", - "_PyLong_CompactValue", - "_PyLong_DigitCount", - "_PyLong_IsCompact", - "_PyLong_IsNegative", - "_PyLong_IsNonNegativeCompact", - "_PyLong_IsZero", - "_PyLong_BothAreCompact", - "_PyCompactLong_Add", - "_PyCompactLong_Multiply", - "_PyCompactLong_Subtract", - "_PyManagedDictPointer_IsValues", - "_PyObject_GC_IS_SHARED", - "_PyObject_GC_IS_TRACKED", - "_PyObject_GC_MAY_BE_TRACKED", - "_PyObject_GC_TRACK", - "_PyObject_GetManagedDict", - "_PyObject_InlineValues", - "_PyObject_IsUniquelyReferenced", - "_PyObject_ManagedDictPointer", - "_PyThreadState_HasStackSpace", - "_PyTuple_FromStackRefStealOnSuccess", - "_PyTuple_ITEMS", - "_PyType_HasFeature", - "_PyType_NewManagedObject", - "_PyUnicode_Equal", - "_PyUnicode_JoinArray", - "_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY", - "_Py_ID", - "_Py_IsImmortal", - "_Py_IsOwnedByCurrentThread", - "_Py_LeaveRecursiveCallPy", - "_Py_LeaveRecursiveCallTstate", - "_Py_NewRef", - "_Py_SINGLETON", - "_Py_STR", - "_Py_TryIncrefCompare", - "_Py_TryIncrefCompareStackRef", - "_Py_atomic_compare_exchange_uint8", - "_Py_atomic_load_ptr_acquire", - "_Py_atomic_load_uintptr_relaxed", - "_Py_set_eval_breaker_bit", - "advance_backoff_counter", - "assert", - "backoff_counter_triggers", - "initial_temperature_backoff_counter", - "JUMP_TO_LABEL", - "restart_backoff_counter", - "_Py_ReachedRecursionLimit", - "PyStackRef_IsTaggedInt", - "PyStackRef_TagInt", - "PyStackRef_UntagInt", - "PyStackRef_IncrementTaggedIntNoOverflow", - "PyStackRef_IsNullOrInt", - "PyStackRef_IsError", - "PyStackRef_IsValid", - "PyStackRef_Wrap", - "PyStackRef_Unwrap", - "_PyLong_CheckExactAndCompact", - "_PyExecutor_FromExit", - "_PyJit_TryInitializeTracing", - "_Py_unset_eval_breaker_bit", - "_Py_set_eval_breaker_bit", - "trigger_backoff_counter", - "_PyThreadState_PopCStackRefSteal", - "doesnt_escape", - "_Py_GatherStats_GetIter", - "_PyStolenTuple_Free", - "PyObject_GC_UnTrack", -) - - -def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None: - error: lexer.Token | None = None - calls = {e.call for e in escapes.values()} - - def visit(stmt: Stmt) -> None: - nonlocal error - if isinstance(stmt, IfStmt) or isinstance(stmt, WhileStmt): - for tkn in stmt.condition: - if tkn in calls: - error = tkn - elif isinstance(stmt, SimpleStmt): - in_if = 0 - tkn_iter = iter(stmt.contents) - for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF", "HANDLE_PENDING_AND_DEOPT_IF", "AT_END_EXIT_IF"): - in_if = 1 - next(tkn_iter) - elif tkn.kind == "LPAREN": - if in_if: - in_if += 1 - elif tkn.kind == "RPAREN": - if in_if: - in_if -= 1 - elif tkn in calls and in_if: - error = tkn - - - instr.block.accept(visit) - if error is not None: - raise analysis_error(f"Escaping call '{error.text} in condition", error) - -def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, EscapingCall]) -> None: - tokens = stmt.contents - for idx, tkn in enumerate(tokens): - try: - next_tkn = tokens[idx+1] - except IndexError: - break - if next_tkn.kind != lexer.LPAREN: - continue - if tkn.kind == lexer.IDENTIFIER: - if tkn.text.upper() == tkn.text: - # simple macro - continue - #if not tkn.text.startswith(("Py", "_Py", "monitor")): - # continue - if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): - # Optimize functions - continue - if tkn.text.endswith("Check"): - continue - if tkn.text.startswith("Py_Is"): - continue - if tkn.text.endswith("CheckExact"): - continue - if tkn.text in NON_ESCAPING_FUNCTIONS: - continue - elif tkn.kind == "RPAREN": - prev = tokens[idx-1] - if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": - #cast - continue - elif tkn.kind != "RBRACKET": - continue - if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): - if len(tokens) <= idx+2: - raise analysis_error("Unexpected end of file", next_tkn) - kills = tokens[idx+2] - if kills.kind != "IDENTIFIER": - raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) - else: - kills = None - result[stmt] = EscapingCall(stmt, tkn, kills) - - -def find_escaping_api_calls(instr: parser.CodeDef) -> dict[SimpleStmt, EscapingCall]: - result: dict[SimpleStmt, EscapingCall] = {} - - def visit(stmt: Stmt) -> None: - if not isinstance(stmt, SimpleStmt): - return - escaping_call_in_simple_stmt(stmt, result) - - instr.block.accept(visit) - check_escaping_calls(instr, result) - return result - - -EXITS = { - "DISPATCH", - "Py_UNREACHABLE", - "DISPATCH_INLINED", - "DISPATCH_GOTO", -} - - -def always_exits(op: parser.CodeDef) -> bool: - depth = 0 - tkn_iter = iter(op.tokens) - for tkn in tkn_iter: - if tkn.kind == "LBRACE": - depth += 1 - elif tkn.kind == "RBRACE": - depth -= 1 - elif depth > 1: - continue - elif tkn.kind == "GOTO" or tkn.kind == "RETURN": - return True - elif tkn.kind == "KEYWORD": - if tkn.text in EXITS: - return True - elif tkn.kind == "IDENTIFIER": - if tkn.text in EXITS: - return True - if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": - next(tkn_iter) # '(' - t = next(tkn_iter) - if t.text in ("true", "1"): - return True - return False - - -def stack_effect_only_peeks(instr: parser.InstDef) -> bool: - stack_inputs = [s for s in instr.inputs if not isinstance(s, parser.CacheEffect)] - if len(stack_inputs) != len(instr.outputs): - return False - if len(stack_inputs) == 0: - return False - return all( - (s.name == other.name and s.size == other.size) - for s, other in zip(stack_inputs, instr.outputs) - ) - - -def stmt_is_simple_exit(stmt: Stmt) -> bool: - if not isinstance(stmt, SimpleStmt): - return False - tokens = stmt.contents - if len(tokens) < 4: - return False - return ( - tokens[0].text in ("ERROR_IF", "DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF") - and - tokens[1].text == "(" - and - tokens[2].text in ("true", "1") - and - tokens[3].text == ")" - ) - - -def stmt_list_escapes(stmts: list[Stmt]) -> bool: - if not stmts: - return False - if stmt_is_simple_exit(stmts[-1]): - return False - for stmt in stmts: - if stmt_escapes(stmt): - return True - return False - - -def stmt_escapes(stmt: Stmt) -> bool: - if isinstance(stmt, BlockStmt): - return stmt_list_escapes(stmt.body) - elif isinstance(stmt, SimpleStmt): - for tkn in stmt.contents: - if tkn.text == "DECREF_INPUTS": - return True - d: dict[SimpleStmt, EscapingCall] = {} - escaping_call_in_simple_stmt(stmt, d) - return bool(d) - elif isinstance(stmt, IfStmt): - if stmt.else_body and stmt_escapes(stmt.else_body): - return True - return stmt_escapes(stmt.body) - elif isinstance(stmt, MacroIfStmt): - if stmt.else_body and stmt_list_escapes(stmt.else_body): - return True - return stmt_list_escapes(stmt.body) - elif isinstance(stmt, ForStmt): - return stmt_escapes(stmt.body) - elif isinstance(stmt, WhileStmt): - return stmt_escapes(stmt.body) - else: - assert False, "Unexpected statement type" - -def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> tuple[bool, int]: - if not stmts: - return False, branches_seen - predict = False - seen = 0 - for st in stmts: - predict_body, seen_body = stmt_has_jump_on_unpredictable_path(st, branches_seen) - predict = predict or predict_body - seen += seen_body - return predict, seen - -def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple[bool, int]: - if isinstance(stmt, BlockStmt): - return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) - elif isinstance(stmt, SimpleStmt): - for tkn in stmt.contents: - if tkn.text == "JUMPBY": - return True, branches_seen - return False, branches_seen - elif isinstance(stmt, IfStmt): - predict, seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) - if stmt.else_body: - predict_else, seen_else = stmt_has_jump_on_unpredictable_path(stmt.else_body, branches_seen) - return predict != predict_else, seen + seen_else + 1 - return predict, seen + 1 - elif isinstance(stmt, MacroIfStmt): - predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) - if stmt.else_body: - predict_else, seen_else = stmt_has_jump_on_unpredictable_path_body(stmt.else_body, branches_seen) - return predict != predict_else, seen + seen_else - return predict, seen - elif isinstance(stmt, ForStmt): - unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) - return unpredictable, branches_seen + 1 - elif isinstance(stmt, WhileStmt): - unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) - return unpredictable, branches_seen + 1 - else: - assert False, f"Unexpected statement type {stmt}" - - -def compute_properties(op: parser.CodeDef) -> Properties: - escaping_calls = find_escaping_api_calls(op) - has_free = ( - variable_used(op, "PyCell_New") - or variable_used(op, "PyCell_GetRef") - or variable_used(op, "PyCell_SetTakeRef") - or variable_used(op, "PyCell_SwapTakeRef") - ) - deopts_if = variable_used(op, "DEOPT_IF") - exits_if = variable_used(op, "EXIT_IF") - exit_if_at_end = variable_used(op, "AT_END_EXIT_IF") - deopts_periodic = variable_used(op, "HANDLE_PENDING_AND_DEOPT_IF") - exits_and_deopts = sum((deopts_if, exits_if, deopts_periodic)) - if exits_and_deopts > 1: - tkn = op.tokens[0] - raise lexer.make_syntax_error( - "Op cannot contain more than one of EXIT_IF, DEOPT_IF and HANDLE_PENDING_AND_DEOPT_IF", - tkn.filename, - tkn.line, - tkn.column, - op.name, - ) - error_with_pop = has_error_with_pop(op) - error_without_pop = has_error_without_pop(op) - escapes = stmt_escapes(op.block) - pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations - no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations - unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(op.block, 0) - unpredictable_jump = False if isinstance(op, parser.LabelDef) else (unpredictable and branches_seen > 0) - return Properties( - escaping_calls=escaping_calls, - escapes=escapes, - error_with_pop=error_with_pop, - error_without_pop=error_without_pop, - deopts=deopts_if, - deopts_periodic=deopts_periodic, - side_exit=exits_if, - side_exit_at_end=exit_if_at_end, - oparg=oparg_used(op), - jumps=variable_used(op, "JUMPBY"), - eval_breaker="CHECK_PERIODIC" in op.name, - needs_this=variable_used(op, "this_instr"), - always_exits=always_exits(op), - sync_sp=variable_used(op, "SYNC_SP"), - uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), - uses_co_names=variable_used(op, "FRAME_CO_NAMES"), - uses_locals=variable_used(op, "GETLOCAL") and not has_free, - uses_opcode=variable_used(op, "opcode"), - has_free=has_free, - pure=pure, - no_save_ip=no_save_ip, - tier=tier_variable(op), - needs_prev=variable_used(op, "prev_instr"), - needs_guard_ip=(isinstance(op, parser.InstDef) - and (unpredictable_jump and "replaced" not in op.annotations)) - or variable_used(op, "LOAD_IP") - or variable_used(op, "DISPATCH_INLINED"), - unpredictable_jump=unpredictable_jump, - records_value=variable_used(op, "RECORD_VALUE") - ) - -def expand(items: list[StackItem], oparg: int) -> list[StackItem]: - # Only replace array item with scalar if no more than one item is an array - index = -1 - for i, item in enumerate(items): - if "oparg" in item.size: - if index >= 0: - return items - index = i - if index < 0: - return items - try: - count = int(eval(items[index].size.replace("oparg", str(oparg)))) - except ValueError: - return items - return items[:index] + [ - StackItem(items[index].name + f"_{i}", "", items[index].peek, items[index].used) for i in range(count) - ] + items[index+1:] - -def scalarize_stack(stack: StackEffect, oparg: int) -> StackEffect: - stack.inputs = expand(stack.inputs, oparg) - stack.outputs = expand(stack.outputs, oparg) - return stack - -def make_uop( - name: str, - op: parser.InstDef, - inputs: list[parser.InputEffect], - uops: dict[str, Uop], -) -> Uop: - result = Uop( - name=name, - context=op.context, - annotations=op.annotations, - stack=analyze_stack(op), - caches=analyze_caches(inputs), - local_stores=find_variable_stores(op), - body=op.block, - properties=compute_properties(op), - ) - for anno in op.annotations: - if anno.startswith("replicate"): - text = anno[10:-1] - start, stop = text.split(":") - result.replicated = range(int(start), int(stop)) - break - else: - return result - for oparg in result.replicated: - name_x = name + "_" + str(oparg) - properties = compute_properties(op) - properties.oparg = False - stack = analyze_stack(op) - if not variable_used(op, "oparg"): - stack = scalarize_stack(stack, oparg) - else: - properties.const_oparg = oparg - rep = Uop( - name=name_x, - context=op.context, - annotations=op.annotations, - stack=stack, - caches=analyze_caches(inputs), - local_stores=find_variable_stores(op), - body=op.block, - properties=properties, - ) - rep.replicates = result - uops[name_x] = rep - - return result - - -def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: - assert op.kind == "op" - if op.name in uops: - if "override" not in op.annotations: - raise override_error( - op.name, op.context, uops[op.name].context, op.tokens[0] - ) - uops[op.name] = make_uop(op.name, op, op.inputs, uops) - - -def add_instruction( - where: lexer.Token, - name: str, - parts: list[Part], - instructions: dict[str, Instruction], -) -> None: - instructions[name] = Instruction(where, name, parts, None) - - -def desugar_inst( - inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop] -) -> None: - assert inst.kind == "inst" - name = inst.name - op_inputs: list[parser.InputEffect] = [] - parts: list[Part] = [] - uop_index = -1 - # Move unused cache entries to the Instruction, removing them from the Uop. - for input in inst.inputs: - if isinstance(input, parser.CacheEffect) and input.name == "unused": - parts.append(Skip(input.size)) - else: - op_inputs.append(input) - if uop_index < 0: - uop_index = len(parts) - # Place holder for the uop. - parts.append(Skip(0)) - uop = make_uop("_" + inst.name, inst, op_inputs, uops) - uop.implicitly_created = True - uops[inst.name] = uop - if uop_index < 0: - parts.append(uop) - else: - parts[uop_index] = uop - add_instruction(inst.first_token, name, parts, instructions) - - -def add_macro( - macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] -) -> None: - parts: list[Part] = [] - # Tracks the last real Uop seen; CacheEffect and flush leave it unchanged. - prev_uop: Uop | None = None - for part in macro.uops: - match part: - case parser.OpName(): - if part.name == "flush": - parts.append(Flush()) - else: - if part.name not in uops: - raise analysis_error( - f"No Uop named {part.name}", macro.tokens[0] - ) - uop = uops[part.name] - if uop.properties.records_value: - # Valid if first real uop, or if the previous real uop - # is specializing (cache slots between them are transparent). - preceding_is_specializing = ( - prev_uop is not None - and "specializing" in prev_uop.annotations - ) - if prev_uop is not None and not preceding_is_specializing: - raise analysis_error( - f"Recording uop {part.name} must be first in macro " - f"or immediately follow a specializing uop", - macro.tokens[0]) - parts.append(uop) - prev_uop = uop # flush and CacheEffect intentionally excluded - case parser.CacheEffect(): - parts.append(Skip(part.size)) - case _: - assert False - assert parts - add_instruction(macro.first_token, macro.name, parts, instructions) - - -def add_family( - pfamily: parser.Family, - instructions: dict[str, Instruction], - families: dict[str, Family], -) -> None: - family = Family( - pfamily.name, - pfamily.size, - [instructions[member_name] for member_name in pfamily.members], - ) - for member in family.members: - member.family = family - # The head of the family is an implicit jump target for DEOPTs - instructions[family.name].is_target = True - families[family.name] = family - - -def add_pseudo( - pseudo: parser.Pseudo, - instructions: dict[str, Instruction], - pseudos: dict[str, PseudoInstruction], -) -> None: - pseudos[pseudo.name] = PseudoInstruction( - pseudo.name, - analyze_stack(pseudo), - [instructions[target] for target in pseudo.targets], - pseudo.as_sequence, - pseudo.flags, - ) - - -def add_label( - label: parser.LabelDef, - labels: dict[str, Label], -) -> None: - properties = compute_properties(label) - labels[label.name] = Label(label.name, label.spilled, label.block, properties) - - -def assign_opcodes( - instructions: dict[str, Instruction], - families: dict[str, Family], - pseudos: dict[str, PseudoInstruction], -) -> tuple[dict[str, int], int, int]: - """Assigns opcodes, then returns the opmap, - have_arg and min_instrumented values""" - instmap: dict[str, int] = {} - - # 0 is reserved for cache entries. This helps debugging. - instmap["CACHE"] = 0 - - # 17 is reserved as it is the initial value for the specializing counter. - # This helps catch cases where we attempt to execute a cache. - instmap["RESERVED"] = 17 - - # 128 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py - instmap["RESUME"] = 128 - - # This is an historical oddity. - instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 - - instmap["INSTRUMENTED_LINE"] = 253 - instmap["ENTER_EXECUTOR"] = 254 - instmap["TRACE_RECORD"] = 255 - - instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] - - specialized: set[str] = set() - no_arg: list[str] = [] - has_arg: list[str] = [] - - for family in families.values(): - specialized.update(inst.name for inst in family.members) - - for inst in instructions.values(): - name = inst.name - if name in specialized: - continue - if name in instrumented: - continue - if inst.properties.oparg: - has_arg.append(name) - else: - no_arg.append(name) - - # Specialized ops appear in their own section - # Instrumented opcodes are at the end of the valid range - min_internal = instmap["RESUME"] + 1 - min_instrumented = 254 - len(instrumented) - assert min_internal + len(specialized) < min_instrumented - - next_opcode = 1 - - def add_instruction(name: str) -> None: - nonlocal next_opcode - if name in instmap: - return # Pre-defined name - while next_opcode in instmap.values(): - next_opcode += 1 - instmap[name] = next_opcode - next_opcode += 1 - - for name in sorted(no_arg): - add_instruction(name) - for name in sorted(has_arg): - add_instruction(name) - # For compatibility - next_opcode = min_internal - for name in sorted(specialized): - add_instruction(name) - next_opcode = min_instrumented - for name in instrumented: - add_instruction(name) - - for name in instructions: - instructions[name].opcode = instmap[name] - - for op, name in enumerate(sorted(pseudos), 256): - instmap[name] = op - pseudos[name].opcode = op - - return instmap, len(no_arg), min_instrumented - - -def get_instruction_size_for_uop(instructions: dict[str, Instruction], uop: Uop) -> int | None: - """Return the size of the instruction that contains the given uop or - `None` if the uop does not contains the `INSTRUCTION_SIZE` macro. - - If there is more than one instruction that contains the uop, - ensure that they all have the same size. - """ - for tkn in uop.body.tokens(): - if tkn.text == "INSTRUCTION_SIZE": - break - else: - return None - - size = None - for inst in instructions.values(): - if uop in inst.parts: - if size is None: - size = inst.size - if size != inst.size: - raise analysis_error( - "All instructions containing a uop with the `INSTRUCTION_SIZE` macro " - f"must have the same size: {size} != {inst.size}", - tkn - ) - if size is None: - raise analysis_error(f"No instruction containing the uop '{uop.name}' was found", tkn) - return size - - -def analyze_forest(forest: list[parser.AstNode]) -> Analysis: - instructions: dict[str, Instruction] = {} - uops: dict[str, Uop] = {} - families: dict[str, Family] = {} - pseudos: dict[str, PseudoInstruction] = {} - labels: dict[str, Label] = {} - for node in forest: - match node: - case parser.InstDef(name): - if node.kind == "inst": - desugar_inst(node, instructions, uops) - else: - assert node.kind == "op" - add_op(node, uops) - case parser.Macro(): - pass - case parser.Family(): - pass - case parser.Pseudo(): - pass - case parser.LabelDef(): - pass - case _: - assert False - for node in forest: - if isinstance(node, parser.Macro): - add_macro(node, instructions, uops) - for node in forest: - match node: - case parser.Family(): - add_family(node, instructions, families) - case parser.Pseudo(): - add_pseudo(node, instructions, pseudos) - case parser.LabelDef(): - add_label(node, labels) - case _: - pass - for uop in uops.values(): - uop.instruction_size = get_instruction_size_for_uop(instructions, uop) - # Special case BINARY_OP_INPLACE_ADD_UNICODE - # BINARY_OP_INPLACE_ADD_UNICODE is not a normal family member, - # as it is the wrong size, but we need it to maintain an - # historical optimization. - if "BINARY_OP_INPLACE_ADD_UNICODE" in instructions: - inst = instructions["BINARY_OP_INPLACE_ADD_UNICODE"] - inst.family = families["BINARY_OP"] - families["BINARY_OP"].members.append(inst) - opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) - return Analysis( - instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented - ) - - -#Simple heuristic for size to avoid too much stencil duplication -def is_large(uop: Uop) -> bool: - return len(list(uop.body.tokens())) > 120 - - -def get_uop_cache_depths(uop: Uop) -> Iterator[tuple[int, int, int]]: - if uop.name == "_SPILL_OR_RELOAD": - for inputs in range(MAX_CACHED_REGISTER+1): - for outputs in range(MAX_CACHED_REGISTER+1): - if inputs != outputs: - yield inputs, outputs, inputs - return - if uop.name in ("_DEOPT", "_HANDLE_PENDING_AND_DEOPT", "_EXIT_TRACE", "_DYNAMIC_EXIT"): - for i in range(MAX_CACHED_REGISTER+1): - yield i, 0, 0 - return - if uop.name in ("_START_EXECUTOR", "_JUMP_TO_TOP", "_COLD_EXIT"): - yield 0, 0, 0 - return - if uop.name == "_ERROR_POP_N": - yield 0, 0, 0 - return - ideal_inputs = 0 - has_array = False - for item in reversed(uop.stack.inputs): - if item.size: - has_array = True - break - ideal_inputs += 1 - ideal_outputs = 0 - for item in reversed(uop.stack.outputs): - if item.size: - has_array = True - break - ideal_outputs += 1 - if ideal_inputs > MAX_CACHED_REGISTER: - ideal_inputs = MAX_CACHED_REGISTER - if ideal_outputs > MAX_CACHED_REGISTER: - ideal_outputs = MAX_CACHED_REGISTER - at_end = uop.properties.sync_sp or uop.properties.side_exit_at_end - exit_depth = ideal_outputs if at_end else ideal_inputs - if uop.properties.escapes or uop.properties.sync_sp or has_array or is_large(uop): - yield ideal_inputs, ideal_outputs, exit_depth - return - for inputs in range(MAX_CACHED_REGISTER + 1): - outputs = ideal_outputs - ideal_inputs + inputs - if outputs < ideal_outputs: - outputs = ideal_outputs - elif outputs > MAX_CACHED_REGISTER: - continue - yield inputs, outputs, outputs if at_end else inputs - - -def analyze_files(filenames: list[str]) -> Analysis: - return analyze_forest(parser.parse_files(filenames)) - - -def dump_analysis(analysis: Analysis) -> None: - print("Uops:") - for u in analysis.uops.values(): - u.dump(" ") - print("Instructions:") - for i in analysis.instructions.values(): - i.dump(" ") - print("Families:") - for f in analysis.families.values(): - f.dump(" ") - print("Pseudos:") - for p in analysis.pseudos.values(): - p.dump(" ") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) < 2: - print("No input") - else: - filenames = sys.argv[1:] - dump_analysis(analyze_files(filenames)) +from dataclasses import dataclass +import itertools +import lexer +import parser +import re +from typing import Optional, Callable, Iterator + +from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, MacroIfStmt + +MAX_CACHED_REGISTER = 3 + +@dataclass +class EscapingCall: + stmt: SimpleStmt + call: lexer.Token + kills: lexer.Token | None + +@dataclass +class Properties: + escaping_calls: dict[SimpleStmt, EscapingCall] + escapes: bool + error_with_pop: bool + error_without_pop: bool + deopts: bool + deopts_periodic: bool + oparg: bool + jumps: bool + eval_breaker: bool + needs_this: bool + always_exits: bool + sync_sp: bool + uses_co_consts: bool + uses_co_names: bool + uses_locals: bool + has_free: bool + side_exit: bool + side_exit_at_end: bool + pure: bool + uses_opcode: bool + needs_guard_ip: bool + unpredictable_jump: bool + records_value: bool + tier: int | None = None + const_oparg: int = -1 + needs_prev: bool = False + no_save_ip: bool = False + + def dump(self, indent: str) -> None: + simple_properties = self.__dict__.copy() + del simple_properties["escaping_calls"] + text = "escaping_calls:\n" + for tkns in self.escaping_calls.values(): + text += f"{indent} {tkns}\n" + text += ", ".join([f"{key}: {value}" for (key, value) in simple_properties.items()]) + print(indent, text, sep="") + + @staticmethod + def from_list(properties: list["Properties"]) -> "Properties": + escaping_calls: dict[SimpleStmt, EscapingCall] = {} + for p in properties: + escaping_calls.update(p.escaping_calls) + return Properties( + escaping_calls=escaping_calls, + escapes = any(p.escapes for p in properties), + error_with_pop=any(p.error_with_pop for p in properties), + error_without_pop=any(p.error_without_pop for p in properties), + deopts=any(p.deopts for p in properties), + deopts_periodic=any(p.deopts_periodic for p in properties), + oparg=any(p.oparg for p in properties), + jumps=any(p.jumps for p in properties), + eval_breaker=any(p.eval_breaker for p in properties), + needs_this=any(p.needs_this for p in properties), + always_exits=any(p.always_exits for p in properties), + sync_sp=any(p.sync_sp for p in properties), + uses_co_consts=any(p.uses_co_consts for p in properties), + uses_co_names=any(p.uses_co_names for p in properties), + uses_locals=any(p.uses_locals for p in properties), + uses_opcode=any(p.uses_opcode for p in properties), + has_free=any(p.has_free for p in properties), + side_exit=any(p.side_exit for p in properties), + side_exit_at_end=any(p.side_exit_at_end for p in properties), + pure=all(p.pure for p in properties), + needs_prev=any(p.needs_prev for p in properties), + no_save_ip=all(p.no_save_ip for p in properties), + needs_guard_ip=any(p.needs_guard_ip for p in properties), + unpredictable_jump=any(p.unpredictable_jump for p in properties), + records_value=any(p.records_value for p in properties), + ) + + @property + def infallible(self) -> bool: + return not self.error_with_pop and not self.error_without_pop + +SKIP_PROPERTIES = Properties( + escaping_calls={}, + escapes=False, + error_with_pop=False, + error_without_pop=False, + deopts=False, + deopts_periodic=False, + oparg=False, + jumps=False, + eval_breaker=False, + needs_this=False, + always_exits=False, + sync_sp=False, + uses_co_consts=False, + uses_co_names=False, + uses_locals=False, + uses_opcode=False, + has_free=False, + side_exit=False, + side_exit_at_end=False, + pure=True, + no_save_ip=False, + needs_guard_ip=False, + unpredictable_jump=False, + records_value=False, +) + + +@dataclass +class Skip: + "Unused cache entry" + size: int + + @property + def name(self) -> str: + return f"unused/{self.size}" + + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES + + +class Flush: + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES + + @property + def name(self) -> str: + return "flush" + + @property + def size(self) -> int: + return 0 + + + + +@dataclass +class StackItem: + name: str + size: str + peek: bool = False + used: bool = False + + def __str__(self) -> str: + size = f"[{self.size}]" if self.size else "" + return f"{self.name}{size} {self.peek}" + + def is_array(self) -> bool: + return self.size != "" + + def get_size(self) -> str: + return self.size if self.size else "1" + + +@dataclass +class StackEffect: + inputs: list[StackItem] + outputs: list[StackItem] + + def __str__(self) -> str: + return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" + + +@dataclass +class CacheEntry: + name: str + size: int + + def __str__(self) -> str: + return f"{self.name}/{self.size}" + + +@dataclass +class Uop: + name: str + context: parser.Context | None + annotations: list[str] + stack: StackEffect + caches: list[CacheEntry] + local_stores: list[lexer.Token] + body: BlockStmt + properties: Properties + _size: int = -1 + implicitly_created: bool = False + replicated = range(0) + replicates: "Uop | None" = None + # Size of the instruction(s), only set for uops containing the INSTRUCTION_SIZE macro + instruction_size: int | None = None + + def dump(self, indent: str) -> None: + print( + indent, self.name, ", ".join(self.annotations) if self.annotations else "" + ) + print(indent, self.stack, ", ".join([str(c) for c in self.caches])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + if self._size < 0: + self._size = sum(c.size for c in self.caches) + return self._size + + def why_not_viable(self) -> str | None: + if self.name == "_SAVE_RETURN_OFFSET": + return None # Adjusts next_instr, but only in tier 1 code + if "INSTRUMENTED" in self.name: + return "is instrumented" + if "replaced" in self.annotations: + return "is replaced" + if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): + return "has tier 1 control flow" + if self.properties.needs_this: + return "uses the 'this_instr' variable" + if len([c for c in self.caches if c.name != "unused"]) > 2: + return "has too many cache entries" + if self.properties.error_with_pop and self.properties.error_without_pop: + return "has both popping and not-popping errors" + return None + + def is_viable(self) -> bool: + return self.why_not_viable() is None + + def is_super(self) -> bool: + for tkn in self.body.tokens(): + if tkn.kind == "IDENTIFIER" and tkn.text == "oparg1": + return True + return False + + +class Label: + + def __init__(self, name: str, spilled: bool, body: BlockStmt, properties: Properties): + self.name = name + self.spilled = spilled + self.body = body + self.properties = properties + + size:int = 0 + local_stores: list[lexer.Token] = [] + instruction_size = None + + def __str__(self) -> str: + return f"label({self.name})" + + +Part = Uop | Skip | Flush +CodeSection = Uop | Label + + +@dataclass +class Instruction: + where: lexer.Token + name: str + parts: list[Part] + _properties: Properties | None + is_target: bool = False + family: Optional["Family"] = None + opcode: int = -1 + + @property + def properties(self) -> Properties: + if self._properties is None: + self._properties = self._compute_properties() + return self._properties + + def _compute_properties(self) -> Properties: + return Properties.from_list([part.properties for part in self.parts]) + + def dump(self, indent: str) -> None: + print(indent, self.name, "=", ", ".join([part.name for part in self.parts])) + self.properties.dump(" " + indent) + + @property + def size(self) -> int: + return 1 + sum(part.size for part in self.parts) + + def is_super(self) -> bool: + if len(self.parts) != 1: + return False + uop = self.parts[0] + if isinstance(uop, Uop): + return uop.is_super() + else: + return False + + +@dataclass +class PseudoInstruction: + name: str + stack: StackEffect + targets: list[Instruction] + as_sequence: bool + flags: list[str] + opcode: int = -1 + + def dump(self, indent: str) -> None: + print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) + + @property + def properties(self) -> Properties: + return Properties.from_list([i.properties for i in self.targets]) + + +@dataclass +class Family: + name: str + size: str + members: list[Instruction] + + def dump(self, indent: str) -> None: + print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) + + +@dataclass +class Analysis: + instructions: dict[str, Instruction] + uops: dict[str, Uop] + families: dict[str, Family] + pseudos: dict[str, PseudoInstruction] + labels: dict[str, Label] + opmap: dict[str, int] + have_arg: int + min_instrumented: int + + +def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: + # To do -- support file and line output + # Construct a SyntaxError instance from message and token + return lexer.make_syntax_error(message, tkn.filename, tkn.line, tkn.column, "") + + +def override_error( + name: str, + context: parser.Context | None, + prev_context: parser.Context | None, + token: lexer.Token, +) -> SyntaxError: + return analysis_error( + f"Duplicate definition of '{name}' @ {context} " + f"previous definition @ {prev_context}", + token, + ) + + +def convert_stack_item( + item: parser.StackEffect, replace_op_arg_1: str | None +) -> StackItem: + return StackItem(item.name, item.size) + +def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]) -> None: + "Unused items cannot be on the stack above used, non-peek items" + seen_unused = False + for item in reversed(stack): + if item.name == "unused": + seen_unused = True + elif item.peek: + break + elif seen_unused: + raise analysis_error(f"Cannot have used input '{item.name}' below an unused value on the stack", input_names[item.name]) + + +def analyze_stack( + op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None +) -> StackEffect: + inputs: list[StackItem] = [ + convert_stack_item(i, replace_op_arg_1) + for i in op.inputs + if isinstance(i, parser.StackEffect) + ] + outputs: list[StackItem] = [ + convert_stack_item(i, replace_op_arg_1) for i in op.outputs + ] + # Mark variables with matching names at the base of the stack as "peek" + modified = False + input_names: dict[str, lexer.Token] = { i.name : i.first_token for i in op.inputs if i.name != "unused" } + for input, output in itertools.zip_longest(inputs, outputs): + if output is None: + pass + elif input is None: + if output.name in input_names: + raise analysis_error( + f"Reuse of variable '{output.name}' at different stack location", + input_names[output.name]) + elif input.name == output.name: + if not modified: + input.peek = output.peek = True + else: + modified = True + if output.name in input_names: + raise analysis_error( + f"Reuse of variable '{output.name}' at different stack location", + input_names[output.name]) + if isinstance(op, parser.InstDef): + output_names = [out.name for out in outputs] + for input in inputs: + if ( + variable_used(op, input.name) + or variable_used(op, "DECREF_INPUTS") + or (not input.peek and input.name in output_names) + ): + input.used = True + for output in outputs: + if variable_used(op, output.name): + output.used = True + check_unused(inputs, input_names) + return StackEffect(inputs, outputs) + + +def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: + caches: list[parser.CacheEffect] = [ + i for i in inputs if isinstance(i, parser.CacheEffect) + ] + if caches: + # Middle entries are allowed to be unused. Check first and last caches. + for index in (0, -1): + cache = caches[index] + if cache.name == "unused": + position = "First" if index == 0 else "Last" + msg = f"{position} cache entry in op is unused. Move to enclosing macro." + raise analysis_error(msg, cache.tokens[0]) + return [CacheEntry(i.name, int(i.size)) for i in caches] + + +def find_variable_stores(node: parser.InstDef) -> list[lexer.Token]: + res: list[lexer.Token] = [] + outnames = { out.name for out in node.outputs } + innames = { out.name for out in node.inputs } + + def find_stores_in_tokens(tokens: list[lexer.Token], callback: Callable[[lexer.Token], None]) -> None: + while tokens and tokens[0].kind == "COMMENT": + tokens = tokens[1:] + if len(tokens) < 4: + return + if tokens[1].kind == "EQUALS": + if tokens[0].kind == "IDENTIFIER": + name = tokens[0].text + if name in outnames or name in innames: + callback(tokens[0]) + #Passing the address of a local is also a definition + for idx, tkn in enumerate(tokens): + if tkn.kind == "AND": + name_tkn = tokens[idx+1] + if name_tkn.text in outnames: + callback(name_tkn) + + def visit(stmt: Stmt) -> None: + if isinstance(stmt, IfStmt): + def error(tkn: lexer.Token) -> None: + raise analysis_error("Cannot define variable in 'if' condition", tkn) + find_stores_in_tokens(stmt.condition, error) + elif isinstance(stmt, SimpleStmt): + find_stores_in_tokens(stmt.contents, res.append) + + node.block.accept(visit) + return res + + +#def analyze_deferred_refs(node: parser.InstDef) -> dict[lexer.Token, str | None]: + #"""Look for PyStackRef_FromPyObjectNew() calls""" + + #def in_frame_push(idx: int) -> bool: + #for tkn in reversed(node.block.tokens[: idx - 1]): + #if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + #return False + #if tkn.kind == "IDENTIFIER" and tkn.text == "_PyFrame_PushUnchecked": + #return True + #return False + + #refs: dict[lexer.Token, str | None] = {} + #for idx, tkn in enumerate(node.block.tokens): + #if tkn.kind != "IDENTIFIER" or tkn.text != "PyStackRef_FromPyObjectNew": + #continue + + #if idx == 0 or node.block.tokens[idx - 1].kind != "EQUALS": + #if in_frame_push(idx): + ## PyStackRef_FromPyObjectNew() is called in _PyFrame_PushUnchecked() + #refs[tkn] = None + #continue + #raise analysis_error("Expected '=' before PyStackRef_FromPyObjectNew", tkn) + + #lhs = find_assignment_target(node, idx - 1) + #if len(lhs) == 0: + #raise analysis_error( + #"PyStackRef_FromPyObjectNew() must be assigned to an output", tkn + #) + + #if lhs[0].kind == "TIMES" or any( + #t.kind == "ARROW" or t.kind == "LBRACKET" for t in lhs[1:] + #): + ## Don't handle: *ptr = ..., ptr->field = ..., or ptr[field] = ... + ## Assume that they are visible to the GC. + #refs[tkn] = None + #continue + + #if len(lhs) != 1 or lhs[0].kind != "IDENTIFIER": + #raise analysis_error( + #"PyStackRef_FromPyObjectNew() must be assigned to an output", tkn + #) + + #name = lhs[0].text + #match = ( + #any(var.name == name for var in node.inputs) + #or any(var.name == name for var in node.outputs) + #) + #if not match: + #raise analysis_error( + #f"PyStackRef_FromPyObjectNew() must be assigned to an input or output, not '{name}'", + #tkn, + #) + + #refs[tkn] = name + + #return refs + + +def variable_used(node: parser.CodeDef, name: str) -> bool: + """Determine whether a variable with a given name is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == name for token in node.block.tokens() + ) + + +def oparg_used(node: parser.CodeDef) -> bool: + """Determine whether `oparg` is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == "oparg" for token in node.tokens + ) + + +def tier_variable(node: parser.CodeDef) -> int | None: + """Determine whether a tier variable is used in a node.""" + if isinstance(node, parser.LabelDef): + return None + for token in node.tokens: + if token.kind == "ANNOTATION": + if token.text == "specializing": + return 1 + if re.fullmatch(r"tier\d", token.text): + return int(token.text[-1]) + return None + + +def has_error_with_pop(op: parser.CodeDef) -> bool: + return ( + variable_used(op, "ERROR_IF") + or variable_used(op, "exception_unwind") + ) + + +def has_error_without_pop(op: parser.CodeDef) -> bool: + return ( + variable_used(op, "ERROR_NO_POP") + or variable_used(op, "exception_unwind") + ) + + +NON_ESCAPING_FUNCTIONS = ( + "PyCFunction_GET_FLAGS", + "PyCFunction_GET_FUNCTION", + "PyCFunction_GET_SELF", + "PyCell_GetRef", + "PyCell_New", + "PyCell_SwapTakeRef", + "PyExceptionInstance_Class", + "PyException_GetCause", + "PyException_GetContext", + "PyException_GetTraceback", + "PyFloat_AS_DOUBLE", + "PyFloat_FromDouble", + "PyFunction_GET_CODE", + "PyFunction_GET_GLOBALS", + "PyList_GET_ITEM", + "PyList_GET_SIZE", + "PyList_SET_ITEM", + "PyLong_AsLong", + "PyLong_FromLong", + "PyLong_FromSsize_t", + "PySlice_New", + "PyStackRef_AsPyObjectBorrow", + "PyStackRef_AsPyObjectNew", + "PyStackRef_FromPyObjectNewMortal", + "PyStackRef_AsPyObjectSteal", + "PyStackRef_Borrow", + "PyStackRef_CLEAR", + "PyStackRef_CLOSE_SPECIALIZED", + "PyStackRef_DUP", + "PyStackRef_False", + "PyStackRef_FromPyObjectBorrow", + "PyStackRef_FromPyObjectNew", + "PyStackRef_FromPyObjectSteal", + "PyStackRef_IsExactly", + "PyStackRef_FromPyObjectStealMortal", + "PyStackRef_IsNone", + "PyStackRef_Is", + "PyStackRef_IsHeapSafe", + "PyStackRef_IsTrue", + "PyStackRef_IsFalse", + "PyStackRef_IsNull", + "PyStackRef_MakeHeapSafe", + "PyStackRef_None", + "PyStackRef_RefcountOnObject", + "PyStackRef_TYPE", + "PyStackRef_True", + "PyTuple_GET_ITEM", + "PyTuple_GET_SIZE", + "PyType_HasFeature", + "PyUnicode_Concat", + "PyUnicode_GET_LENGTH", + "PyUnicode_READ_CHAR", + "PyUnicode_IS_COMPACT_ASCII", + "PyUnicode_1BYTE_DATA", + "Py_ARRAY_LENGTH", + "Py_FatalError", + "Py_INCREF", + "Py_IS_TYPE", + "Py_NewRef", + "Py_REFCNT", + "Py_SIZE", + "Py_TYPE", + "Py_UNREACHABLE", + "Py_Unicode_GET_LENGTH", + "_PyCode_CODE", + "_PyDictValues_AddToInsertionOrder", + "_PyErr_Occurred", + "_PyFrame_GetBytecode", + "_PyFrame_GetCode", + "_PyFrame_IsIncomplete", + "_PyFrame_PushUnchecked", + "_PyFrame_SetStackPointer", + "_PyFrame_StackPush", + "_PyFunction_SetVersion", + "_PyGen_GetGeneratorFromFrame", + "gen_try_set_executing", + "_PyInterpreterState_GET", + "_PyList_AppendTakeRef", + "_PyList_ITEMS", + "_PyLong_CompactValue", + "_PyLong_DigitCount", + "_PyLong_IsCompact", + "_PyLong_IsNegative", + "_PyLong_IsNonNegativeCompact", + "_PyLong_IsZero", + "_PyLong_BothAreCompact", + "_PyCompactLong_Add", + "_PyCompactLong_Multiply", + "_PyCompactLong_Subtract", + "_PyManagedDictPointer_IsValues", + "_PyObject_GC_IS_SHARED", + "_PyObject_GC_IS_TRACKED", + "_PyObject_GC_MAY_BE_TRACKED", + "_PyObject_GC_TRACK", + "_PyObject_GetManagedDict", + "_PyObject_InlineValues", + "_PyObject_IsUniquelyReferenced", + "_PyObject_ManagedDictPointer", + "_PyThreadState_HasStackSpace", + "_PyTuple_FromStackRefStealOnSuccess", + "_PyTuple_ITEMS", + "_PyType_HasFeature", + "_PyType_NewManagedObject", + "_PyUnicode_Equal", + "_PyUnicode_JoinArray", + "_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY", + "_Py_ID", + "_Py_IsImmortal", + "_Py_IsOwnedByCurrentThread", + "_Py_LeaveRecursiveCallPy", + "_Py_LeaveRecursiveCallTstate", + "_Py_NewRef", + "_Py_SINGLETON", + "_Py_STR", + "_Py_TryIncrefCompare", + "_Py_TryIncrefCompareStackRef", + "_Py_atomic_compare_exchange_uint8", + "_Py_atomic_load_ptr_acquire", + "_Py_atomic_load_uintptr_relaxed", + "_Py_set_eval_breaker_bit", + "advance_backoff_counter", + "assert", + "backoff_counter_triggers", + "initial_temperature_backoff_counter", + "JUMP_TO_LABEL", + "restart_backoff_counter", + "_Py_ReachedRecursionLimit", + "PyStackRef_IsTaggedInt", + "PyStackRef_TagInt", + "PyStackRef_UntagInt", + "PyStackRef_IncrementTaggedIntNoOverflow", + "PyStackRef_IsNullOrInt", + "PyStackRef_IsError", + "PyStackRef_IsValid", + "PyStackRef_Wrap", + "PyStackRef_Unwrap", + "_PyLong_CheckExactAndCompact", + "_PyExecutor_FromExit", + "_PyJit_TryInitializeTracing", + "_Py_unset_eval_breaker_bit", + "_Py_set_eval_breaker_bit", + "trigger_backoff_counter", + "_PyThreadState_PopCStackRefSteal", + "doesnt_escape", + "_Py_GatherStats_GetIter", + "_PyStolenTuple_Free", + "PyObject_GC_UnTrack", +) + + +def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None: + error: lexer.Token | None = None + calls = {e.call for e in escapes.values()} + + def visit(stmt: Stmt) -> None: + nonlocal error + if isinstance(stmt, IfStmt) or isinstance(stmt, WhileStmt): + for tkn in stmt.condition: + if tkn in calls: + error = tkn + elif isinstance(stmt, SimpleStmt): + in_if = 0 + tkn_iter = iter(stmt.contents) + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF", "HANDLE_PENDING_AND_DEOPT_IF", "AT_END_EXIT_IF"): + in_if = 1 + next(tkn_iter) + elif tkn.kind == "LPAREN": + if in_if: + in_if += 1 + elif tkn.kind == "RPAREN": + if in_if: + in_if -= 1 + elif tkn in calls and in_if: + error = tkn + + + instr.block.accept(visit) + if error is not None: + raise analysis_error(f"Escaping call '{error.text} in condition", error) + +def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, EscapingCall]) -> None: + tokens = stmt.contents + for idx, tkn in enumerate(tokens): + try: + next_tkn = tokens[idx+1] + except IndexError: + break + if next_tkn.kind != lexer.LPAREN: + continue + if tkn.kind == lexer.IDENTIFIER: + if tkn.text.upper() == tkn.text: + # simple macro + continue + #if not tkn.text.startswith(("Py", "_Py", "monitor")): + # continue + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): + # Optimize functions + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + elif tkn.kind == "RPAREN": + prev = tokens[idx-1] + if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": + #cast + continue + elif tkn.kind != "RBRACKET": + continue + if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): + if len(tokens) <= idx+2: + raise analysis_error("Unexpected end of file", next_tkn) + kills = tokens[idx+2] + if kills.kind != "IDENTIFIER": + raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) + else: + kills = None + result[stmt] = EscapingCall(stmt, tkn, kills) + + +def find_escaping_api_calls(instr: parser.CodeDef) -> dict[SimpleStmt, EscapingCall]: + result: dict[SimpleStmt, EscapingCall] = {} + + def visit(stmt: Stmt) -> None: + if not isinstance(stmt, SimpleStmt): + return + escaping_call_in_simple_stmt(stmt, result) + + instr.block.accept(visit) + check_escaping_calls(instr, result) + return result + + +EXITS = { + "DISPATCH", + "Py_UNREACHABLE", + "DISPATCH_INLINED", + "DISPATCH_GOTO", +} + + +def always_exits(op: parser.CodeDef) -> bool: + depth = 0 + tkn_iter = iter(op.tokens) + for tkn in tkn_iter: + if tkn.kind == "LBRACE": + depth += 1 + elif tkn.kind == "RBRACE": + depth -= 1 + elif depth > 1: + continue + elif tkn.kind == "GOTO" or tkn.kind == "RETURN": + return True + elif tkn.kind == "KEYWORD": + if tkn.text in EXITS: + return True + elif tkn.kind == "IDENTIFIER": + if tkn.text in EXITS: + return True + if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": + next(tkn_iter) # '(' + t = next(tkn_iter) + if t.text in ("true", "1"): + return True + return False + + +def stack_effect_only_peeks(instr: parser.InstDef) -> bool: + stack_inputs = [s for s in instr.inputs if not isinstance(s, parser.CacheEffect)] + if len(stack_inputs) != len(instr.outputs): + return False + if len(stack_inputs) == 0: + return False + return all( + (s.name == other.name and s.size == other.size) + for s, other in zip(stack_inputs, instr.outputs) + ) + + +def stmt_is_simple_exit(stmt: Stmt) -> bool: + if not isinstance(stmt, SimpleStmt): + return False + tokens = stmt.contents + if len(tokens) < 4: + return False + return ( + tokens[0].text in ("ERROR_IF", "DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF") + and + tokens[1].text == "(" + and + tokens[2].text in ("true", "1") + and + tokens[3].text == ")" + ) + + +def stmt_list_escapes(stmts: list[Stmt]) -> bool: + if not stmts: + return False + if stmt_is_simple_exit(stmts[-1]): + return False + for stmt in stmts: + if stmt_escapes(stmt): + return True + return False + + +def stmt_escapes(stmt: Stmt) -> bool: + if isinstance(stmt, BlockStmt): + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "DECREF_INPUTS": + return True + d: dict[SimpleStmt, EscapingCall] = {} + escaping_call_in_simple_stmt(stmt, d) + return bool(d) + elif isinstance(stmt, IfStmt): + if stmt.else_body and stmt_escapes(stmt.else_body): + return True + return stmt_escapes(stmt.body) + elif isinstance(stmt, MacroIfStmt): + if stmt.else_body and stmt_list_escapes(stmt.else_body): + return True + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, ForStmt): + return stmt_escapes(stmt.body) + elif isinstance(stmt, WhileStmt): + return stmt_escapes(stmt.body) + else: + assert False, "Unexpected statement type" + +def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> tuple[bool, int]: + if not stmts: + return False, branches_seen + predict = False + seen = 0 + for st in stmts: + predict_body, seen_body = stmt_has_jump_on_unpredictable_path(st, branches_seen) + predict = predict or predict_body + seen += seen_body + return predict, seen + +def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple[bool, int]: + if isinstance(stmt, BlockStmt): + return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "JUMPBY": + return True, branches_seen + return False, branches_seen + elif isinstance(stmt, IfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + 1 + return predict, seen + 1 + elif isinstance(stmt, MacroIfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path_body(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + return predict, seen + elif isinstance(stmt, ForStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + elif isinstance(stmt, WhileStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + else: + assert False, f"Unexpected statement type {stmt}" + + +def compute_properties(op: parser.CodeDef) -> Properties: + escaping_calls = find_escaping_api_calls(op) + has_free = ( + variable_used(op, "PyCell_New") + or variable_used(op, "PyCell_GetRef") + or variable_used(op, "PyCell_SetTakeRef") + or variable_used(op, "PyCell_SwapTakeRef") + ) + deopts_if = variable_used(op, "DEOPT_IF") + exits_if = variable_used(op, "EXIT_IF") + exit_if_at_end = variable_used(op, "AT_END_EXIT_IF") + deopts_periodic = variable_used(op, "HANDLE_PENDING_AND_DEOPT_IF") + exits_and_deopts = sum((deopts_if, exits_if, deopts_periodic)) + if exits_and_deopts > 1: + tkn = op.tokens[0] + raise lexer.make_syntax_error( + "Op cannot contain more than one of EXIT_IF, DEOPT_IF and HANDLE_PENDING_AND_DEOPT_IF", + tkn.filename, + tkn.line, + tkn.column, + op.name, + ) + error_with_pop = has_error_with_pop(op) + error_without_pop = has_error_without_pop(op) + escapes = stmt_escapes(op.block) + pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations + no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(op.block, 0) + unpredictable_jump = False if isinstance(op, parser.LabelDef) else (unpredictable and branches_seen > 0) + return Properties( + escaping_calls=escaping_calls, + escapes=escapes, + error_with_pop=error_with_pop, + error_without_pop=error_without_pop, + deopts=deopts_if, + deopts_periodic=deopts_periodic, + side_exit=exits_if, + side_exit_at_end=exit_if_at_end, + oparg=oparg_used(op), + jumps=variable_used(op, "JUMPBY"), + eval_breaker="CHECK_PERIODIC" in op.name, + needs_this=variable_used(op, "this_instr"), + always_exits=always_exits(op), + sync_sp=variable_used(op, "SYNC_SP"), + uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), + uses_co_names=variable_used(op, "FRAME_CO_NAMES"), + uses_locals=variable_used(op, "GETLOCAL") and not has_free, + uses_opcode=variable_used(op, "opcode"), + has_free=has_free, + pure=pure, + no_save_ip=no_save_ip, + tier=tier_variable(op), + needs_prev=variable_used(op, "prev_instr"), + needs_guard_ip=(isinstance(op, parser.InstDef) + and (unpredictable_jump and "replaced" not in op.annotations)) + or variable_used(op, "LOAD_IP") + or variable_used(op, "DISPATCH_INLINED"), + unpredictable_jump=unpredictable_jump, + records_value=variable_used(op, "RECORD_VALUE") + ) + +def expand(items: list[StackItem], oparg: int) -> list[StackItem]: + # Only replace array item with scalar if no more than one item is an array + index = -1 + for i, item in enumerate(items): + if "oparg" in item.size: + if index >= 0: + return items + index = i + if index < 0: + return items + try: + count = int(eval(items[index].size.replace("oparg", str(oparg)))) + except ValueError: + return items + return items[:index] + [ + StackItem(items[index].name + f"_{i}", "", items[index].peek, items[index].used) for i in range(count) + ] + items[index+1:] + +def scalarize_stack(stack: StackEffect, oparg: int) -> StackEffect: + stack.inputs = expand(stack.inputs, oparg) + stack.outputs = expand(stack.outputs, oparg) + return stack + +def make_uop( + name: str, + op: parser.InstDef, + inputs: list[parser.InputEffect], + uops: dict[str, Uop], +) -> Uop: + result = Uop( + name=name, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + local_stores=find_variable_stores(op), + body=op.block, + properties=compute_properties(op), + ) + for anno in op.annotations: + if anno.startswith("replicate"): + text = anno[10:-1] + start, stop = text.split(":") + result.replicated = range(int(start), int(stop)) + break + else: + return result + for oparg in result.replicated: + name_x = name + "_" + str(oparg) + properties = compute_properties(op) + properties.oparg = False + stack = analyze_stack(op) + if not variable_used(op, "oparg"): + stack = scalarize_stack(stack, oparg) + else: + properties.const_oparg = oparg + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=stack, + caches=analyze_caches(inputs), + local_stores=find_variable_stores(op), + body=op.block, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + + return result + + +def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: + assert op.kind == "op" + if op.name in uops: + if "override" not in op.annotations: + raise override_error( + op.name, op.context, uops[op.name].context, op.tokens[0] + ) + uops[op.name] = make_uop(op.name, op, op.inputs, uops) + + +def add_instruction( + where: lexer.Token, + name: str, + parts: list[Part], + instructions: dict[str, Instruction], +) -> None: + instructions[name] = Instruction(where, name, parts, None) + + +def desugar_inst( + inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + assert inst.kind == "inst" + name = inst.name + op_inputs: list[parser.InputEffect] = [] + parts: list[Part] = [] + uop_index = -1 + # Move unused cache entries to the Instruction, removing them from the Uop. + for input in inst.inputs: + if isinstance(input, parser.CacheEffect) and input.name == "unused": + parts.append(Skip(input.size)) + else: + op_inputs.append(input) + if uop_index < 0: + uop_index = len(parts) + # Place holder for the uop. + parts.append(Skip(0)) + uop = make_uop("_" + inst.name, inst, op_inputs, uops) + uop.implicitly_created = True + uops[inst.name] = uop + if uop_index < 0: + parts.append(uop) + else: + parts[uop_index] = uop + add_instruction(inst.first_token, name, parts, instructions) + + +def add_macro( + macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: + parts: list[Part] = [] + # Tracks the last "plain" uop (neither specializing nor recording). + # CacheEffect, flush, specializing, and recording uops all leave it + # unchanged, so prev_uop being non-None is sufficient to mean + # "a disqualifying uop has been seen before this recording uop". + prev_uop: Uop | None = None + for part in macro.uops: + match part: + case parser.OpName(): + if part.name == "flush": + parts.append(Flush()) + else: + if part.name not in uops: + raise analysis_error( + f"No Uop named {part.name}", macro.tokens[0] + ) + uop = uops[part.name] + if uop.properties.records_value and prev_uop is not None: + raise analysis_error( + f"Recording uop {part.name} must be first in macro " + f"or immediately follow a specializing uop", + macro.tokens[0]) + parts.append(uop) + # Only plain worker uops set prev_uop; specializing and + # recording uops are excluded so the check above stays simple. + if not uop.properties.records_value and "specializing" not in uop.annotations: + prev_uop = uop + case parser.CacheEffect(): + parts.append(Skip(part.size)) + case _: + assert False + assert parts + add_instruction(macro.first_token, macro.name, parts, instructions) + + + +def add_family( + pfamily: parser.Family, + instructions: dict[str, Instruction], + families: dict[str, Family], +) -> None: + family = Family( + pfamily.name, + pfamily.size, + [instructions[member_name] for member_name in pfamily.members], + ) + for member in family.members: + member.family = family + # The head of the family is an implicit jump target for DEOPTs + instructions[family.name].is_target = True + families[family.name] = family + + +def add_pseudo( + pseudo: parser.Pseudo, + instructions: dict[str, Instruction], + pseudos: dict[str, PseudoInstruction], +) -> None: + pseudos[pseudo.name] = PseudoInstruction( + pseudo.name, + analyze_stack(pseudo), + [instructions[target] for target in pseudo.targets], + pseudo.as_sequence, + pseudo.flags, + ) + + +def add_label( + label: parser.LabelDef, + labels: dict[str, Label], +) -> None: + properties = compute_properties(label) + labels[label.name] = Label(label.name, label.spilled, label.block, properties) + + +def assign_opcodes( + instructions: dict[str, Instruction], + families: dict[str, Family], + pseudos: dict[str, PseudoInstruction], +) -> tuple[dict[str, int], int, int]: + """Assigns opcodes, then returns the opmap, + have_arg and min_instrumented values""" + instmap: dict[str, int] = {} + + # 0 is reserved for cache entries. This helps debugging. + instmap["CACHE"] = 0 + + # 17 is reserved as it is the initial value for the specializing counter. + # This helps catch cases where we attempt to execute a cache. + instmap["RESERVED"] = 17 + + # 128 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py + instmap["RESUME"] = 128 + + # This is an historical oddity. + instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 + + instmap["INSTRUMENTED_LINE"] = 253 + instmap["ENTER_EXECUTOR"] = 254 + instmap["TRACE_RECORD"] = 255 + + instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] + + specialized: set[str] = set() + no_arg: list[str] = [] + has_arg: list[str] = [] + + for family in families.values(): + specialized.update(inst.name for inst in family.members) + + for inst in instructions.values(): + name = inst.name + if name in specialized: + continue + if name in instrumented: + continue + if inst.properties.oparg: + has_arg.append(name) + else: + no_arg.append(name) + + # Specialized ops appear in their own section + # Instrumented opcodes are at the end of the valid range + min_internal = instmap["RESUME"] + 1 + min_instrumented = 254 - len(instrumented) + assert min_internal + len(specialized) < min_instrumented + + next_opcode = 1 + + def add_instruction(name: str) -> None: + nonlocal next_opcode + if name in instmap: + return # Pre-defined name + while next_opcode in instmap.values(): + next_opcode += 1 + instmap[name] = next_opcode + next_opcode += 1 + + for name in sorted(no_arg): + add_instruction(name) + for name in sorted(has_arg): + add_instruction(name) + # For compatibility + next_opcode = min_internal + for name in sorted(specialized): + add_instruction(name) + next_opcode = min_instrumented + for name in instrumented: + add_instruction(name) + + for name in instructions: + instructions[name].opcode = instmap[name] + + for op, name in enumerate(sorted(pseudos), 256): + instmap[name] = op + pseudos[name].opcode = op + + return instmap, len(no_arg), min_instrumented + + +def get_instruction_size_for_uop(instructions: dict[str, Instruction], uop: Uop) -> int | None: + """Return the size of the instruction that contains the given uop or + `None` if the uop does not contains the `INSTRUCTION_SIZE` macro. + + If there is more than one instruction that contains the uop, + ensure that they all have the same size. + """ + for tkn in uop.body.tokens(): + if tkn.text == "INSTRUCTION_SIZE": + break + else: + return None + + size = None + for inst in instructions.values(): + if uop in inst.parts: + if size is None: + size = inst.size + if size != inst.size: + raise analysis_error( + "All instructions containing a uop with the `INSTRUCTION_SIZE` macro " + f"must have the same size: {size} != {inst.size}", + tkn + ) + if size is None: + raise analysis_error(f"No instruction containing the uop '{uop.name}' was found", tkn) + return size + + +def analyze_forest(forest: list[parser.AstNode]) -> Analysis: + instructions: dict[str, Instruction] = {} + uops: dict[str, Uop] = {} + families: dict[str, Family] = {} + pseudos: dict[str, PseudoInstruction] = {} + labels: dict[str, Label] = {} + for node in forest: + match node: + case parser.InstDef(name): + if node.kind == "inst": + desugar_inst(node, instructions, uops) + else: + assert node.kind == "op" + add_op(node, uops) + case parser.Macro(): + pass + case parser.Family(): + pass + case parser.Pseudo(): + pass + case parser.LabelDef(): + pass + case _: + assert False + for node in forest: + if isinstance(node, parser.Macro): + add_macro(node, instructions, uops) + for node in forest: + match node: + case parser.Family(): + add_family(node, instructions, families) + case parser.Pseudo(): + add_pseudo(node, instructions, pseudos) + case parser.LabelDef(): + add_label(node, labels) + case _: + pass + for uop in uops.values(): + uop.instruction_size = get_instruction_size_for_uop(instructions, uop) + # Special case BINARY_OP_INPLACE_ADD_UNICODE + # BINARY_OP_INPLACE_ADD_UNICODE is not a normal family member, + # as it is the wrong size, but we need it to maintain an + # historical optimization. + if "BINARY_OP_INPLACE_ADD_UNICODE" in instructions: + inst = instructions["BINARY_OP_INPLACE_ADD_UNICODE"] + inst.family = families["BINARY_OP"] + families["BINARY_OP"].members.append(inst) + opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) + return Analysis( + instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented + ) + + +#Simple heuristic for size to avoid too much stencil duplication +def is_large(uop: Uop) -> bool: + return len(list(uop.body.tokens())) > 120 + + +def get_uop_cache_depths(uop: Uop) -> Iterator[tuple[int, int, int]]: + if uop.name == "_SPILL_OR_RELOAD": + for inputs in range(MAX_CACHED_REGISTER+1): + for outputs in range(MAX_CACHED_REGISTER+1): + if inputs != outputs: + yield inputs, outputs, inputs + return + if uop.name in ("_DEOPT", "_HANDLE_PENDING_AND_DEOPT", "_EXIT_TRACE", "_DYNAMIC_EXIT"): + for i in range(MAX_CACHED_REGISTER+1): + yield i, 0, 0 + return + if uop.name in ("_START_EXECUTOR", "_JUMP_TO_TOP", "_COLD_EXIT"): + yield 0, 0, 0 + return + if uop.name == "_ERROR_POP_N": + yield 0, 0, 0 + return + ideal_inputs = 0 + has_array = False + for item in reversed(uop.stack.inputs): + if item.size: + has_array = True + break + ideal_inputs += 1 + ideal_outputs = 0 + for item in reversed(uop.stack.outputs): + if item.size: + has_array = True + break + ideal_outputs += 1 + if ideal_inputs > MAX_CACHED_REGISTER: + ideal_inputs = MAX_CACHED_REGISTER + if ideal_outputs > MAX_CACHED_REGISTER: + ideal_outputs = MAX_CACHED_REGISTER + at_end = uop.properties.sync_sp or uop.properties.side_exit_at_end + exit_depth = ideal_outputs if at_end else ideal_inputs + if uop.properties.escapes or uop.properties.sync_sp or has_array or is_large(uop): + yield ideal_inputs, ideal_outputs, exit_depth + return + for inputs in range(MAX_CACHED_REGISTER + 1): + outputs = ideal_outputs - ideal_inputs + inputs + if outputs < ideal_outputs: + outputs = ideal_outputs + elif outputs > MAX_CACHED_REGISTER: + continue + yield inputs, outputs, outputs if at_end else inputs + + +def analyze_files(filenames: list[str]) -> Analysis: + return analyze_forest(parser.parse_files(filenames)) + + +def dump_analysis(analysis: Analysis) -> None: + print("Uops:") + for u in analysis.uops.values(): + u.dump(" ") + print("Instructions:") + for i in analysis.instructions.values(): + i.dump(" ") + print("Families:") + for f in analysis.families.values(): + f.dump(" ") + print("Pseudos:") + for p in analysis.pseudos.values(): + p.dump(" ") + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("No input") + else: + filenames = sys.argv[1:] + dump_analysis(analyze_files(filenames))