Step in/step over support for IPython. Fixes #869

Author: fabioz

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info_regular.py

@@ -53,6 +53,8 @@ class PyDBAdditionalThreadInfo(object):
         # of the last request for a given thread and pydev_smart_parent_offset/pydev_smart_child_offset relies on it).
         'pydev_smart_step_into_variants',
         'target_id_to_smart_step_into_variant',
+
+        'pydev_use_scoped_step_frame',
     ]
     # ENDIF
 
@@ -90,6 +92,18 @@ def __init__(self):
         self.pydev_smart_step_into_variants = ()
         self.target_id_to_smart_step_into_variant = {}
 
+        # Flag to indicate ipython use-case where each line will be executed as a call/line/return
+        # in a new new frame but in practice we want to consider each new frame as if it was all
+        # part of the same frame.
+        #
+        # In practice this means that a step over shouldn't revert to a step in and we need some
+        # special logic to know when we should stop in a step over as we need to consider 2
+        # different frames as being equal if they're logically the continuation of a frame
+        # being executed by ipython line by line.
+        #
+        # See: https://github.com/microsoft/debugpy/issues/869#issuecomment-1132141003
+        self.pydev_use_scoped_step_frame = False
+
     def get_topmost_frame(self, thread):
         '''
         Gets the topmost frame for the given thread. Note that it may be None

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py

@@ -255,11 +255,6 @@ def as_int_in_env(env_key, default):
 # If not specified, uses default heuristic to determine if it should be loaded.
 USE_CYTHON_FLAG = os.getenv('PYDEVD_USE_CYTHON')
 
-# Use to disable loading the lib to set tracing to all threads (default is using heuristics based on where we're running).
-LOAD_NATIVE_LIB_FLAG = os.getenv('PYDEVD_LOAD_NATIVE_LIB', '').lower()
-
-LOG_TIME = os.getenv('PYDEVD_LOG_TIME', 'true').lower() in ENV_TRUE_LOWER_VALUES
-
 if USE_CYTHON_FLAG is not None:
     USE_CYTHON_FLAG = USE_CYTHON_FLAG.lower()
     if USE_CYTHON_FLAG not in ENV_TRUE_LOWER_VALUES and USE_CYTHON_FLAG not in ENV_FALSE_LOWER_VALUES:
@@ -270,6 +265,26 @@ def as_int_in_env(env_key, default):
     if not CYTHON_SUPPORTED:
         USE_CYTHON_FLAG = 'no'
 
+# If true in env, forces frame eval to be used (raises error if not available).
+# If false in env, disables it.
+# If not specified, uses default heuristic to determine if it should be loaded.
+PYDEVD_USE_FRAME_EVAL = os.getenv('PYDEVD_USE_FRAME_EVAL', '').lower()
+
+PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING = is_true_in_env('PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING')
+
+# If specified in PYDEVD_IPYTHON_CONTEXT it must be a string with the basename
+# and then the name of 2 methods in which the evaluate is done.
+PYDEVD_IPYTHON_CONTEXT = ('interactiveshell.py', 'run_code', 'run_ast_nodes')
+_ipython_ctx = os.getenv('PYDEVD_IPYTHON_CONTEXT')
+if _ipython_ctx:
+    PYDEVD_IPYTHON_CONTEXT = tuple(x.strip() for x in _ipython_ctx.split(','))
+    assert len(PYDEVD_IPYTHON_CONTEXT) == 3, 'Invalid PYDEVD_IPYTHON_CONTEXT: %s' % (_ipython_ctx,)
+
+# Use to disable loading the lib to set tracing to all threads (default is using heuristics based on where we're running).
+LOAD_NATIVE_LIB_FLAG = os.getenv('PYDEVD_LOAD_NATIVE_LIB', '').lower()
+
+LOG_TIME = os.getenv('PYDEVD_LOG_TIME', 'true').lower() in ENV_TRUE_LOWER_VALUES
+
 SHOW_COMPILE_CYTHON_COMMAND_LINE = is_true_in_env('PYDEVD_SHOW_COMPILE_CYTHON_COMMAND_LINE')
 
 LOAD_VALUES_ASYNC = is_true_in_env('PYDEVD_LOAD_VALUES_ASYNC')

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.c

@@ -24,3 +24,4 @@ cdef class PyDBAdditionalThreadInfo:
     cdef public int pydev_smart_child_offset
     cdef public tuple pydev_smart_step_into_variants
     cdef public dict target_id_to_smart_step_into_variant
+    cdef public bint pydev_use_scoped_step_frame

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.pxd

@@ -59,6 +59,8 @@ cdef class PyDBAdditionalThreadInfo:
 #         # of the last request for a given thread and pydev_smart_parent_offset/pydev_smart_child_offset relies on it).
 #         'pydev_smart_step_into_variants',
 #         'target_id_to_smart_step_into_variant',
+# 
+#         'pydev_use_scoped_step_frame',
 #     ]
     # ENDIF
 
@@ -96,6 +98,18 @@ cdef class PyDBAdditionalThreadInfo:
         self.pydev_smart_step_into_variants = ()
         self.target_id_to_smart_step_into_variant = {}
 
+        # Flag to indicate ipython use-case where each line will be executed as a call/line/return
+        # in a new new frame but in practice we want to consider each new frame as if it was all
+        # part of the same frame.
+        #
+        # In practice this means that a step over shouldn't revert to a step in and we need some
+        # special logic to know when we should stop in a step over as we need to consider 2
+        # different frames as being equal if they're logically the continuation of a frame
+        # being executed by ipython line by line.
+        #
+        # See: https://github.com/microsoft/debugpy/issues/869#issuecomment-1132141003
+        self.pydev_use_scoped_step_frame = False
+
     def get_topmost_frame(self, thread):
         '''
         Gets the topmost frame for the given thread. Note that it may be None
@@ -150,7 +164,7 @@ import re
 from _pydev_bundle import pydev_log
 from _pydevd_bundle import pydevd_dont_trace
 from _pydevd_bundle.pydevd_constants import (RETURN_VALUES_DICT, NO_FTRACE,
-    EXCEPTION_TYPE_HANDLED, EXCEPTION_TYPE_USER_UNHANDLED)
+    EXCEPTION_TYPE_HANDLED, EXCEPTION_TYPE_USER_UNHANDLED, PYDEVD_IPYTHON_CONTEXT)
 from _pydevd_bundle.pydevd_frame_utils import add_exception_to_frame, just_raised, remove_exception_from_frame, ignore_exception_trace
 from _pydevd_bundle.pydevd_utils import get_clsname_for_code
 from pydevd_file_utils import get_abs_path_real_path_and_base_from_frame
@@ -657,6 +671,31 @@ cdef class PyDBFrame:
 
         return f
 
+    # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
+    cdef _is_same_frame(self, target_frame, current_frame):
+        cdef PyDBAdditionalThreadInfo info;
+    # ELSE
+#     def _is_same_frame(self, target_frame, current_frame):
+    # ENDIF
+        if target_frame is current_frame:
+            return True
+
+        info = self._args[2]
+        if info.pydev_use_scoped_step_frame:
+            # If using scoped step we don't check the target, we just need to check
+            # if the current matches the same heuristic where the target was defined.
+            if target_frame is not None and current_frame is not None:
+                if target_frame.f_code.co_filename == current_frame.f_code.co_filename:
+                    # The co_name may be different (it may include the line number), but
+                    # the filename must still be the same.
+                    f = current_frame.f_back
+                    if f is not None and f.f_code.co_name == PYDEVD_IPYTHON_CONTEXT[1]:
+                        f = f.f_back
+                        if f is not None and f.f_code.co_name == PYDEVD_IPYTHON_CONTEXT[2]:
+                            return True
+
+        return False
+
     # IFDEF CYTHON -- DONT EDIT THIS FILE (it is automatically generated)
     cpdef trace_dispatch(self, frame, str event, arg):
         cdef tuple abs_path_canonical_path_and_base;
@@ -772,7 +811,13 @@ cdef class PyDBFrame:
                         # Solving this may not be trivial as we'd need to put a scope in the step
                         # in, but we may have to do it anyways to have a step in which doesn't end
                         # up in asyncio).
-                        if stop_frame is frame:
+                        #
+                        # Note2: we don't revert to a step in if we're doing scoped stepping
+                        # (because on scoped stepping we're always receiving a call/line/return
+                        # event for each line in ipython, so, we can't revert to step in on return
+                        # as the return shouldn't mean that we've actually completed executing a
+                        # frame in this case).
+                        if stop_frame is frame and not info.pydev_use_scoped_step_frame:
                             if step_cmd in (108, 159, 107, 144):
                                 f = self._get_unfiltered_back_frame(main_debugger, frame)
                                 if f is not None:
@@ -809,7 +854,7 @@ cdef class PyDBFrame:
                     # event == 'call' or event == 'c_XXX'
                     return self.trace_dispatch
 
-            else:
+            else:  # Not coroutine nor generator
                 if event == 'line':
                     is_line = True
                     is_call = False
@@ -828,7 +873,12 @@ cdef class PyDBFrame:
                     # to make a step in or step over at that location).
                     # Note: this is especially troublesome when we're skipping code with the
                     # @DontTrace comment.
-                    if stop_frame is frame and is_return and step_cmd in (108, 109, 159, 160, 128):
+                    if (
+                            stop_frame is frame and
+                            not info.pydev_use_scoped_step_frame and is_return and
+                            step_cmd in (108, 109, 159, 160, 128)
+                        ):
+
                         if step_cmd in (108, 109, 128):
                             info.pydev_step_cmd = 107
                         else:
@@ -876,7 +926,7 @@ cdef class PyDBFrame:
                     if step_cmd == -1:
                         can_skip = True
 
-                    elif step_cmd in (108, 109, 159, 160) and stop_frame is not frame:
+                    elif step_cmd in (108, 109, 159, 160) and not self._is_same_frame(stop_frame, frame):
                         can_skip = True
 
                     elif step_cmd == 128 and (
@@ -896,7 +946,7 @@ cdef class PyDBFrame:
                     elif step_cmd == 206:
                         f = frame
                         while f is not None:
-                            if f is stop_frame:
+                            if self._is_same_frame(stop_frame, f):
                                 break
                             f = f.f_back
                         else:
@@ -907,7 +957,7 @@ cdef class PyDBFrame:
                                 main_debugger.has_plugin_line_breaks or main_debugger.has_plugin_exception_breaks):
                             can_skip = plugin_manager.can_skip(main_debugger, frame)
 
-                        if can_skip and main_debugger.show_return_values and info.pydev_step_cmd in (108, 159) and frame.f_back is stop_frame:
+                        if can_skip and main_debugger.show_return_values and info.pydev_step_cmd in (108, 159) and self._is_same_frame(stop_frame, frame.f_back):
                             # trace function for showing return values after step over
                             can_skip = False
 
@@ -1006,7 +1056,7 @@ cdef class PyDBFrame:
                     breakpoint = breakpoints_for_file[line]
                     new_frame = frame
                     stop = True
-                    if step_cmd in (108, 159) and (stop_frame is frame and is_line):
+                    if step_cmd in (108, 159) and (self._is_same_frame(stop_frame, frame) and is_line):
                         stop = False  # we don't stop on breakpoint if we have to stop by step-over (it will be processed later)
                 elif plugin_manager is not None and main_debugger.has_plugin_line_breaks:
                     result = plugin_manager.get_breakpoint(main_debugger, self, frame, event, self._args)
@@ -1050,8 +1100,8 @@ cdef class PyDBFrame:
 
                 if main_debugger.show_return_values:
                     if is_return and (
-                            (info.pydev_step_cmd in (108, 159, 128) and (frame.f_back is stop_frame)) or
-                            (info.pydev_step_cmd in (109, 160) and (frame is stop_frame)) or
+                            (info.pydev_step_cmd in (108, 159, 128) and (self._is_same_frame(stop_frame, frame.f_back))) or
+                            (info.pydev_step_cmd in (109, 160) and (self._is_same_frame(stop_frame, frame))) or
                             (info.pydev_step_cmd in (107, 206)) or
                             (
                                 info.pydev_step_cmd == 144
@@ -1115,12 +1165,36 @@ cdef class PyDBFrame:
                 elif step_cmd in (107, 144, 206):
                     force_check_project_scope = step_cmd == 144
                     if is_line:
-                        if force_check_project_scope or main_debugger.is_files_filter_enabled:
-                            stop = not main_debugger.apply_files_filter(frame, frame.f_code.co_filename, force_check_project_scope)
+                        if not info.pydev_use_scoped_step_frame:
+                            if force_check_project_scope or main_debugger.is_files_filter_enabled:
+                                stop = not main_debugger.apply_files_filter(frame, frame.f_code.co_filename, force_check_project_scope)
+                            else:
+                                stop = True
                         else:
-                            stop = True
+                            # We can only stop inside the ipython call.
+                            filename = frame.f_code.co_filename
+                            if filename.endswith('.pyc'):
+                                filename = filename[:-1]
+
+                            if not filename.endswith(PYDEVD_IPYTHON_CONTEXT[0]):
+                                f = frame.f_back
+                                while f is not None:
+                                    if f.f_code.co_name == PYDEVD_IPYTHON_CONTEXT[1]:
+                                        f2 = f.f_back
+                                        if f2 is not None and f2.f_code.co_name == PYDEVD_IPYTHON_CONTEXT[2]:
+                                            pydev_log.debug('Stop inside ipython call')
+                                            stop = True
+                                            break
+                                    f = f.f_back
+
+                                del f
+
+                            if not stop:
+                                # In scoped mode if step in didn't work in this context it won't work
+                                # afterwards anyways.
+                                return None if is_call else NO_FTRACE
 
-                    elif is_return and frame.f_back is not None:
+                    elif is_return and frame.f_back is not None and not info.pydev_use_scoped_step_frame:
                         if main_debugger.get_file_type(frame.f_back) == main_debugger.PYDEV_FILE:
                             stop = False
                         else:
@@ -1141,7 +1215,7 @@ cdef class PyDBFrame:
                             # i.e.: Check if we're stepping into the proper context.
                             f = frame
                             while f is not None:
-                                if f is stop_frame:
+                                if self._is_same_frame(stop_frame, f):
                                     break
                                 f = f.f_back
                             else:
@@ -1156,7 +1230,7 @@ cdef class PyDBFrame:
                     # Note: when dealing with a step over my code it's the same as a step over (the
                     # difference is that when we return from a frame in one we go to regular step
                     # into and in the other we go to a step into my code).
-                    stop = stop_frame is frame and is_line
+                    stop = self._is_same_frame(stop_frame, frame) and is_line
                     # Note: don't stop on a return for step over, only for line events
                     # i.e.: don't stop in: (stop_frame is frame.f_back and is_return) as we'd stop twice in that line.
 
@@ -1168,11 +1242,11 @@ cdef class PyDBFrame:
                 elif step_cmd == 128:
                     stop = False
                     back = frame.f_back
-                    if stop_frame is frame and is_return:
+                    if self._is_same_frame(stop_frame, frame) and is_return:
                         # We're exiting the smart step into initial frame (so, we probably didn't find our target).
                         stop = True
 
-                    elif stop_frame is back and is_line:
+                    elif self._is_same_frame(stop_frame, back) and is_line:
                         if info.pydev_smart_child_offset != -1:
                             # i.e.: in this case, we're not interested in the pause in the parent, rather
                             # we're interested in the pause in the child (when the parent is at the proper place).
@@ -1203,7 +1277,7 @@ cdef class PyDBFrame:
                             # not be the case next time either, so, disable tracing for this frame.
                             return None if is_call else NO_FTRACE
 
-                    elif back is not None and stop_frame is back.f_back and is_line:
+                    elif back is not None and self._is_same_frame(stop_frame, back.f_back) and is_line:
                         # Ok, we have to track 2 stops at this point, the parent and the child offset.
                         # This happens when handling a step into which targets a function inside a list comprehension
                         # or generator (in which case an intermediary frame is created due to an internal function call).
@@ -1237,7 +1311,7 @@ cdef class PyDBFrame:
                             return None if is_call else NO_FTRACE
 
                 elif step_cmd in (109, 160):
-                    stop = is_return and stop_frame is frame
+                    stop = is_return and self._is_same_frame(stop_frame, frame)
 
                 else:
                     stop = False