From 9c4c42746fd8e6f6ec3518fa84f0706c9e1a989f Mon Sep 17 00:00:00 2001 From: xFrednet Date: Wed, 8 Apr 2026 17:22:55 +0200 Subject: [PATCH 1/8] TRegion: Very basic TRegion type --- Include/internal/pycore_immutability.h | 2 + Lib/immutable.py | 1 + Makefile.pre.in | 1 + Modules/_immutablemodule.c | 8 ++++ Objects/tracingregionobject.c | 57 ++++++++++++++++++++++++++ PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 ++ PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 ++ 9 files changed, 77 insertions(+) create mode 100644 Objects/tracingregionobject.c diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h index 8e4d32b78527a6..4e79803b50fd12 100644 --- a/Include/internal/pycore_immutability.h +++ b/Include/internal/pycore_immutability.h @@ -8,6 +8,8 @@ extern "C" { # error "Py_BUILD_CORE must be defined to include this header" #endif +PyAPI_DATA(PyTypeObject) _PyTracingRegion_Type; + struct _Py_immutability_state { int late_init_done; struct _Py_hashtable_t *shallow_immutable_types; diff --git a/Lib/immutable.py b/Lib/immutable.py index e1c00152f94bbd..40cce9c93cbefb 100644 --- a/Lib/immutable.py +++ b/Lib/immutable.py @@ -21,6 +21,7 @@ FREEZABLE_PROXY = _c.FREEZABLE_PROXY InterpreterLocal = _c.InterpreterLocal SharedField = _c.SharedField +TracingRegion = _c.TracingRegion # FIXME(immutable): For the longest time we used the name `isfrozen` # without the underscore. This keeps the function name for now, but diff --git a/Makefile.pre.in b/Makefile.pre.in index 572a784546b60f..1b55ebe01a2a85 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -555,6 +555,7 @@ OBJECT_OBJS= \ Objects/sliceobject.o \ Objects/structseq.o \ Objects/templateobject.o \ + Objects/tracingregionobject.o \ Objects/tupleobject.o \ Objects/typeobject.o \ Objects/typevarobject.o \ diff --git a/Modules/_immutablemodule.c b/Modules/_immutablemodule.c index 94ea460c340526..7a237c6e1a7a8c 100644 --- a/Modules/_immutablemodule.c +++ b/Modules/_immutablemodule.c @@ -650,6 +650,14 @@ immutable_exec(PyObject *module) { return -1; } + if (PyModule_AddType(module, &_PyTracingRegion_Type) != 0) { + return -1; + } + if (_PyImmutability_SetFreezable( + (PyObject*)&_PyTracingRegion_Type, _Py_FREEZABLE_YES) < 0) { + return -1; + } + if (PyModule_AddIntConstant(module, "FREEZABLE_YES", _Py_FREEZABLE_YES) != 0) { return -1; diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c new file mode 100644 index 00000000000000..251cf5a0cb3250 --- /dev/null +++ b/Objects/tracingregionobject.c @@ -0,0 +1,57 @@ +#include "Python.h" +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() + +typedef struct { + PyObject_HEAD + PyObject *dict; +} TracingRegionObject; + + +static int +TracingRegion_traverse(TracingRegionObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + return 0; +} + +static int +TracingRegion_clear(TracingRegionObject *self) +{ + Py_CLEAR(self->dict); + return 0; +} + +static void +TracingRegion_dealloc(TracingRegionObject *self) +{ + PyObject_GC_UnTrack(self); + TracingRegion_clear(self); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +TracingRegion_init(TracingRegionObject *self, PyObject *args, PyObject *kwds) +{ + return 0; +} + +static PyMemberDef TracingRegion_members[] = { + {"__dict__", _Py_T_OBJECT, offsetof(TracingRegionObject, dict), Py_READONLY}, + {NULL} +}; + +PyTypeObject _PyTracingRegion_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "TracingRegion", + .tp_basicsize = sizeof(TracingRegionObject), + .tp_dealloc = (destructor)TracingRegion_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .tp_traverse = (traverseproc)TracingRegion_traverse, + .tp_clear = (inquiry)TracingRegion_clear, + .tp_members = TracingRegion_members, + .tp_dictoffset = offsetof(TracingRegionObject, dict), + .tp_init = (initproc)TracingRegion_init, + .tp_new = PyType_GenericNew, + .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, +}; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 3702e4e9998718..c2514ce954c902 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -161,6 +161,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 0b968eba5b977b..ebb3c7a469b443 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -478,6 +478,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d6ce53bbea2824..b61f2669d4974d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -559,6 +559,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d5351a82741a0f..61fe05d775a7b5 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1273,6 +1273,9 @@ Objects + + Objects + Objects From 78e5fe4795ee64b106809034e4eea856f52873d3 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Wed, 8 Apr 2026 18:08:38 +0200 Subject: [PATCH 2/8] Copy and pasta the needed bits --- Objects/tracingregionobject.c | 240 +++++++++++++++++++++++++++++++++- 1 file changed, 234 insertions(+), 6 deletions(-) diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index 251cf5a0cb3250..4b50276bef7214 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -1,13 +1,228 @@ #include "Python.h" +#include "pycore_interp.h" #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_descrobject.h" + +// #define REGION_TRACING + +#ifdef REGION_TRACING +#define if_trace(...) __VA_ARGS__ +#define trace_arg(arg) , (Py_uintptr_t)(arg) +#define trace(msg, region, ...) \ + do { \ + printf(msg "\n", (Py_region_t)(region) __VA_OPT__(,) __VA_ARGS__); \ + } while(0) +#define trace_lrc(...) trace(__VA_ARGS__) +#else +#define if_trace(...) +#define trace_arg(...) +#define trace(...) +#define trace_lrc(...) +#endif + +/* Macro that jumps to error, if the expression `x` does not succeed. */ +#define SUCCEEDS(x) do { int r = (x); if (r != 0) goto error; } while (0) + +// ################################################################### +// Copied from gc.c +// ################################################################### + +#ifndef Py_GIL_DISABLED +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV + +static inline void +gc_set_old_space(PyGC_Head *g, int space) +{ + assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); + g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; + g->_gc_next |= space; +} + +static inline void +gc_list_init(PyGC_Head *list) +{ + // List header must not have flags. + // We can assign pointer by simple cast. + list->_gc_prev = (uintptr_t)list; + list->_gc_next = (uintptr_t)list; +} + +static void +gc_list_move(PyGC_Head *node, PyGC_Head *list) +{ + /* Unlink from current list. */ + PyGC_Head *from_prev = GC_PREV(node); + PyGC_Head *from_next = GC_NEXT(node); + _PyGCHead_SET_NEXT(from_prev, from_next); + _PyGCHead_SET_PREV(from_next, from_prev); + + /* Relink at end of new list. */ + // list must not have flags. So we can skip macros. + PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; + _PyGCHead_SET_PREV(node, to_prev); + _PyGCHead_SET_NEXT(to_prev, node); + list->_gc_prev = (uintptr_t)node; + _PyGCHead_SET_NEXT(node, list); +} + +static inline int +gc_list_is_empty(PyGC_Head *list) +{ + return (list->_gc_next == (uintptr_t)list); +} + +static void +gc_list_merge(PyGC_Head *from, PyGC_Head *to) +{ + assert(from != to); + if (!gc_list_is_empty(from)) { + PyGC_Head *to_tail = GC_PREV(to); + PyGC_Head *from_head = GC_NEXT(from); + PyGC_Head *from_tail = GC_PREV(from); + assert(from_head != from); + assert(from_tail != from); + + _PyGCHead_SET_NEXT(to_tail, from_head); + _PyGCHead_SET_PREV(from_head, to_tail); + + _PyGCHead_SET_NEXT(from_tail, to); + _PyGCHead_SET_PREV(to, from_tail); + } + gc_list_init(from); +} + +static struct _gc_runtime_state* +get_gc_state(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->gc; +} + +static inline void +gc_clear_collecting(PyGC_Head *g) +{ + g->_gc_prev &= ~_PyGC_PREV_MASK_COLLECTING; +} + +#elif // Py_GIL_DISABLED +#error "We need GIL" +#endif + +// ################################################################### +// Copied from regions-main +// ################################################################### + +typedef enum { + Py_MOVABLE_YES = 0, + Py_MOVABLE_NO = 1, + Py_MOVABLE_FREEZE = 2, +} movable_status; + +movable_status get_movable_status(PyObject *obj) { + // FIXME(regions): xFrednet: Currently it's not possible to set + // the movability per object. This instead returns the default + // movability for objects. Note that some shallow immutable objects + // will not return freeze as their movability. + + // Immortal object have no real RC, this makes it infeasible to have them + // in a region and dynamically track their ownership. Immortal objects are + // intended to be immutable in Python, so it should be safe to implicitly + // freeze them. + if (_Py_IsImmortal(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Immutable objects don't need to be moved + if (_Py_IsImmutable(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Types are a pain for regions since it's likely that objects of one type may + // end up in multiple regions, requiring the type to be frozen. Types also + // have a lot of reference pointing to them. Let's hope there is no need to + // keep them freezable + if (PyType_Check(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Module objects are also complicated. Freezing them should turn most modules + // into proxys which should make them mostly usable. + if (PyModule_Check(obj)) { + return Py_MOVABLE_FREEZE; + } + + // Functions are a mess as well, making the entire system reachable. Freezing + // them should again just magically make most things work + if (PyFunction_Check(obj)) { + return Py_MOVABLE_FREEZE; + } + + // CWrappers can't really be owned, but need some special handling since + // interpreters could still race on their RC. Solution, throw them in the + // freezer + if (PyCFunction_Check(obj) + || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) + || Py_IS_TYPE(obj, &PyWrapperDescr_Type) + ) { + return Py_MOVABLE_FREEZE; + } + + // Freezing or moving these objects is... complicated. In some cases it is + // possible but more hassle than it's probably worth. For not we mark them + // all as unmovable. + if (PyFrame_Check(obj) + || PyGen_CheckExact(obj) + || PyCoro_CheckExact(obj) + || PyAsyncGen_CheckExact(obj) + || PyAsyncGenASend_CheckExact(obj) + ) { + return Py_MOVABLE_NO; + } + + // Exceptions don't hold anything obviously problematic preventing them + // from being moved into a region. The actual problem is that the runtime + // stores references to them and that these are already emitted on an + // error path. Moving them into a region could add more problems. + // We should discuss how to handle these, maybe freezing is the correct + // approach? + if (PyExceptionInstance_Check(obj)) { + return Py_MOVABLE_NO; + } + + // For now, we define all other objects as movable by default. (Surely + // this will not backfire) + return Py_MOVABLE_YES; +} + +// ################################################################### +// Tracing Impl +// ################################################################### + +typedef struct { + Py_ssize_t objs; + Py_ssize_t incoming_refs; +} trace_res; + +static trace_res trace_object(PyObject* obj) { + trace_res res = { + .objs = 1, + .incoming_refs = 2, + }; + + return res; +} + +// ################################################################### +// Region Object +// ################################################################### typedef struct { PyObject_HEAD PyObject *dict; } TracingRegionObject; - static int TracingRegion_traverse(TracingRegionObject *self, visitproc visit, void *arg) { @@ -30,12 +245,23 @@ TracingRegion_dealloc(TracingRegionObject *self) Py_TYPE(self)->tp_free((PyObject *)self); } -static int -TracingRegion_init(TracingRegionObject *self, PyObject *args, PyObject *kwds) -{ - return 0; +static PyObject* TracingRegion_trace(PyObject *op) { + trace_res res = trace_object(op); + + PyObject *t = Py_BuildValue("(ii)", res.objs, res.incoming_refs); + if (t == NULL) { + return NULL; // propagate Python exception + } + + return t; } +static PyMethodDef TracingRegion_methods[] = { + {"trace", _PyCFunction_CAST(TracingRegion_trace), METH_NOARGS, + "This traces the region and returns the number of incoming references"}, + {NULL, NULL} /* sentinel */ +}; + static PyMemberDef TracingRegion_members[] = { {"__dict__", _Py_T_OBJECT, offsetof(TracingRegionObject, dict), Py_READONLY}, {NULL} @@ -50,8 +276,10 @@ PyTypeObject _PyTracingRegion_Type = { .tp_traverse = (traverseproc)TracingRegion_traverse, .tp_clear = (inquiry)TracingRegion_clear, .tp_members = TracingRegion_members, + .tp_methods = TracingRegion_methods, .tp_dictoffset = offsetof(TracingRegionObject, dict), - .tp_init = (initproc)TracingRegion_init, .tp_new = PyType_GenericNew, .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, }; + + From 8f11ad1e22dd692c4283ec809016ea170ed41148 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Wed, 15 Apr 2026 14:22:00 +0200 Subject: [PATCH 3/8] TracingRegions a working prototype --- Objects/tracingregionobject.c | 335 ++++++++++++++++++++++++++++++++-- 1 file changed, 317 insertions(+), 18 deletions(-) diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index 4b50276bef7214..a0602f6bb5b60d 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -9,16 +9,14 @@ #ifdef REGION_TRACING #define if_trace(...) __VA_ARGS__ #define trace_arg(arg) , (Py_uintptr_t)(arg) -#define trace(msg, region, ...) \ +#define trace(msg, ...) \ do { \ - printf(msg "\n", (Py_region_t)(region) __VA_OPT__(,) __VA_ARGS__); \ + printf(msg "\n" __VA_OPT__(,) __VA_ARGS__); \ } while(0) -#define trace_lrc(...) trace(__VA_ARGS__) #else #define if_trace(...) #define trace_arg(...) #define trace(...) -#define trace_lrc(...) #endif /* Macro that jumps to error, if the expression `x` does not succeed. */ @@ -114,6 +112,24 @@ gc_clear_collecting(PyGC_Head *g) // Copied from regions-main // ################################################################### +static PyObject* list_pop(PyObject* s){ + PyObject* item; + Py_ssize_t size = PyList_Size(s); + if(size == 0){ + return NULL; + } + item = PyList_GetItem(s, size - 1); + if(item == NULL){ + return NULL; + } + // This should never fail, since we shrink the size + if(PyList_SetSlice(s, size - 1, size, NULL)){ + Py_DECREF(item); + return NULL; + } + return item; +} + typedef enum { Py_MOVABLE_YES = 0, Py_MOVABLE_NO = 1, @@ -196,24 +212,298 @@ movable_status get_movable_status(PyObject *obj) { return Py_MOVABLE_YES; } +// This uses the given arguments to create and throw a `RegionError` +static void throw_region_error( + const char *format_str, const char *tp_name, + PyObject* src, PyObject* tgt) +{ + // Don't stomp existing exception + PyThreadState *tstate = PyThreadState_Get(); + if (_PyErr_Occurred(tstate)) { + return; + } + + PyErr_Format(PyExc_RuntimeError, format_str, tp_name); + + // Set source and target fields + // Get the current exception (should be a RuntimeError) + PyObject *exc = PyErr_GetRaisedException(); + assert(exc && PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_RuntimeError)); + + // Add 'source' and 'target' attributes to the exception + PyObject_SetAttr(exc, &_Py_ID(source), src ? src : Py_None); + PyObject_SetAttr(exc, &_Py_ID(target), tgt ? tgt : Py_None); + + PyErr_SetRaisedException((PyObject*)exc); +} + +// Wrapper around tp_traverse that also visits the type object. +static int +traverse_via_tp_traverse(PyObject *obj, visitproc visit, void *state) +{ + PyTypeObject *tp = Py_TYPE(obj); + + // Visit the type with traverse + traverseproc traverse = tp->tp_traverse; + if (traverse != NULL) { + int err = traverse(obj, visit, state); + if (err) { + return err; + } + } + + + // Most `tp_traverse` don't visit the type even though they should. + // Here it won't hurt to potentially visit it twice, since types + // are non-movable but will be frozen. + return visit((PyObject *)Py_TYPE(obj), state); +} + +// Returns the appropriate traversal function for reaching all references +// from an object. Prefers tp_reachable, falls back to tp_traverse wrapped +// to also visit the type. Emits a warning once per type on fallback. +static traverseproc +get_reachable_proc(PyTypeObject *tp) +{ + if (tp->tp_reachable != NULL) { + return tp->tp_reachable; + } + + if (tp->tp_traverse != NULL) { + PySys_FormatStderr( + "regions: type '%.100s' has tp_traverse but no tp_reachable\n", + tp->tp_name); + } else { + PySys_FormatStderr( + "regions: type '%.100s' has no tp_traverse and no tp_reachable\n", + tp->tp_name); + } + + // Always return the wrapper; even when tp_traverse is NULL, the wrapper + // will still visit the type object which tp_reachable is expected to do. + return traverse_via_tp_traverse; +} + // ################################################################### // Tracing Impl // ################################################################### +static void +gc_list_dissolve(PyGC_Head *list) { + struct _gc_runtime_state* gc_state = get_gc_state(); + // Use `old[0]` here, we are setting the visited space to 0 in add_visited_set(). + gc_list_merge(list, &(gc_state->old[0].head)); +} + +typedef struct { + /// A list of all visited objects + _Py_hashtable_t *visited; + /// The number of refs coming into this object graph + Py_ssize_t external_rc; + // This is set if an object was frozen and the trace needs to restart to be valid + bool restart; + // The GC list used for this trace + PyGC_Head* gc_list; + // The source of the reference, this is used for error reporting + PyObject *src; + // List of pending objects that are not GC + PyObject *pending; +} trace_state; + +static void trace_state_destroy(trace_state* state, bool dissolve_gc) { + if (state->visited) { + _Py_hashtable_destroy(state->visited); + state->visited = NULL; + } + if (state->pending) { + Py_DECREF(state->pending); + state->pending = NULL; + } + + if (dissolve_gc) { + gc_list_dissolve(state->gc_list); + } +} +static int trace_state_init(trace_state* state, PyGC_Head *gc_list) { + assert(gc_list_is_empty(gc_list)); + + state->visited = NULL; + state->pending = NULL; + + state->visited = _Py_hashtable_new( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (state->visited == NULL) { + goto error; + } + + state->pending = PyList_New(0); + if (state->pending == NULL) { + goto error; + } + + state->external_rc = 0; + state->restart = false; + state->gc_list = gc_list; + state->src = NULL; + + return 0; +error: + trace_state_destroy(state, false); + return -1; +} + typedef struct { Py_ssize_t objs; Py_ssize_t incoming_refs; -} trace_res; +} trace_result; + +const int TRACE_RES_ERR = -1; +const int TRACE_RES_DONE = 0; +const int TRACE_RES_RESTART = 1; + +static int _move_obj(PyObject* obj, trace_state* state) { + // Check the movability of the object: + movable_status status = get_movable_status(obj); + switch (status) { + case Py_MOVABLE_YES: + break; + case Py_MOVABLE_NO: + trace(" - %p is not movable", obj); + throw_region_error( + "Instances of type '%s' are not movable", Py_TYPE(obj)->tp_name, + state->src, obj); + return TRACE_RES_ERR; + case Py_MOVABLE_FREEZE: + // Freeze the object, this can invalidate our `external_rc`, we restart after this trace + trace(" - freezing %p", obj); + if (_PyImmutability_Freeze(obj)) { + return TRACE_RES_ERR; + } + + state->restart = true; + return 0; + default: + assert(false); + break; + } + + // Move the object + Py_ssize_t lrc_change = Py_REFCNT(obj); + if (state->src != NULL) { + // -1 for the reference we just followed + lrc_change -= 1; + } + trace(" - moving %p; LRC += %zd", obj, lrc_change); + state->external_rc += lrc_change; + + if (_Py_hashtable_set(state->visited, obj, obj) == -1) { + return -1; + } + + // This makes sure the object is removed from the local GC list. + if (PyObject_IS_GC(obj) && PyObject_GC_IsTracked(obj)) { + // This flag may be set if the region is constructed as part of + // a finalizer. If the flag remains set, for an object removed + // from its GC list bad things can happen. + gc_clear_collecting(_Py_AS_GC(obj)); + // Clearing the space flag makes it easy to merge this list back + // into the local GC lists + gc_set_old_space(_Py_AS_GC(obj), 0); + gc_list_move(_Py_AS_GC(obj), state->gc_list); + } + + if (PyList_Append(state->pending, obj)) { + return -1; + } + + return 0; +} + +static int _trace_visit(PyObject* obj, trace_state* state) { + // References to immutable objects are allowed + if (_PyImmutability_CanViewAsImmutable(obj)) { + assert(_Py_IsImmutable(obj)); + return 0; + } -static trace_res trace_object(PyObject* obj) { - trace_res res = { - .objs = 1, - .incoming_refs = 2, - }; + // Check if the object is already part of the region + if (_Py_hashtable_get(state->visited, (void*)obj)) { + trace(" - Internal reference to %p; LRC -= 1", obj); + state->external_rc -= 1; + return 0; + } + + return _move_obj(obj, state); +} + +static int _trace_once(PyObject* obj, trace_result* result, PyGC_Head *gc_list) { + trace(" - starting trace from %p", obj); + int res = TRACE_RES_DONE; + + // Make sure gc_list is valid + PyGC_Head local_gc_list; + bool dissolve_gc = false; + if (gc_list == NULL) { + gc_list_init(&local_gc_list); + gc_list = &local_gc_list; + dissolve_gc = true; + } + // init the trace state + trace_state state; + if (trace_state_init(&state, gc_list)) { + return TRACE_RES_ERR; + } + + SUCCEEDS(_move_obj(obj, &state)); + + while (PyList_GET_SIZE(state.pending) > 0) { + // Find the next pending item: + PyObject *item = list_pop(state.pending); + + // Traverse item + state.src = item; + trace(" - traversing %p", item); + traverseproc proc = get_reachable_proc(Py_TYPE(item)); + SUCCEEDS(proc(item, (visitproc)_trace_visit, (void*)&state)); + } + + if (state.restart) { + res = TRACE_RES_RESTART; + } + + goto finally; +error: + dissolve_gc = true; + res = TRACE_RES_ERR; +finally: + result->incoming_refs = state.external_rc; + result->objs = _Py_hashtable_len(state.visited); + trace_state_destroy(&state, dissolve_gc); return res; } +static int trace_object(PyObject* obj, trace_result* result, PyGC_Head *gc_list) { + const int TRIES = 2; + trace("Starting trace for %p", obj); + for (int i = 0; i < TRIES; i++) { + // Reset trace + result->objs = 0; + result->incoming_refs = 0; + + // Trace object + int res = _trace_once(obj, result, gc_list); + if (res == TRACE_RES_RESTART) { + trace("- restarting trace for %p", obj); + continue; + } + return res; + } + + return TRACE_RES_DONE; +} + // ################################################################### // Region Object // ################################################################### @@ -221,34 +511,42 @@ static trace_res trace_object(PyObject* obj) { typedef struct { PyObject_HEAD PyObject *dict; + // The GC list containing all objects, used during transfer + PyGC_Head gc_list; } TracingRegionObject; static int -TracingRegion_traverse(TracingRegionObject *self, visitproc visit, void *arg) -{ +TracingRegion_init(TracingRegionObject *self, PyObject *args, PyObject *kwargs) { + gc_list_init(&self->gc_list); + return 0; +} + +static int +TracingRegion_traverse(TracingRegionObject *self, visitproc visit, void *arg) { Py_VISIT(self->dict); return 0; } static int -TracingRegion_clear(TracingRegionObject *self) -{ +TracingRegion_clear(TracingRegionObject *self) { Py_CLEAR(self->dict); return 0; } static void -TracingRegion_dealloc(TracingRegionObject *self) -{ +TracingRegion_dealloc(TracingRegionObject *self) { PyObject_GC_UnTrack(self); TracingRegion_clear(self); Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject* TracingRegion_trace(PyObject *op) { - trace_res res = trace_object(op); + trace_result result; + if (trace_object(op, &result, NULL)) { + return NULL; // propagate Python exception + } - PyObject *t = Py_BuildValue("(ii)", res.objs, res.incoming_refs); + PyObject *t = Py_BuildValue("(ii)", result.objs, result.incoming_refs); if (t == NULL) { return NULL; // propagate Python exception } @@ -278,6 +576,7 @@ PyTypeObject _PyTracingRegion_Type = { .tp_members = TracingRegion_members, .tp_methods = TracingRegion_methods, .tp_dictoffset = offsetof(TracingRegionObject, dict), + .tp_init = (initproc)TracingRegion_init, .tp_new = PyType_GenericNew, .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, }; From cf0cacb0b383bbedaa0e9155487fd0b9dea9d79c Mon Sep 17 00:00:00 2001 From: xFrednet Date: Thu, 16 Apr 2026 11:53:33 +0200 Subject: [PATCH 4/8] TracingRegions C-interface for closing --- Include/internal/pycore_immutability.h | 2 + Lib/test/test_freeze/test_tracing_region.py | 76 ++++++++++++++++++ Objects/tracingregionobject.c | 88 +++++++++++++++------ 3 files changed, 143 insertions(+), 23 deletions(-) create mode 100644 Lib/test/test_freeze/test_tracing_region.py diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h index 4e79803b50fd12..e696fe0542598e 100644 --- a/Include/internal/pycore_immutability.h +++ b/Include/internal/pycore_immutability.h @@ -9,6 +9,8 @@ extern "C" { #endif PyAPI_DATA(PyTypeObject) _PyTracingRegion_Type; +PyAPI_FUNC(int) _PyTracingRegion_Close(PyObject* region); +PyAPI_FUNC(int) _PyTracingRegion_Open(PyObject* region); struct _Py_immutability_state { int late_init_done; diff --git a/Lib/test/test_freeze/test_tracing_region.py b/Lib/test/test_freeze/test_tracing_region.py new file mode 100644 index 00000000000000..3c430122e91aec --- /dev/null +++ b/Lib/test/test_freeze/test_tracing_region.py @@ -0,0 +1,76 @@ +import sys +import unittest +from immutable import freeze, is_frozen, freezable +from immutable import TracingRegion as Region + +class TestTraceRefs(unittest.TestCase): + def test_trace(self): + @freezable + class A: + pass + + r = Region() + r.a = A() + r.b = A() + r.c = A() + + _, base_refs = r.trace() + + a = r.a + _, ref_count = r.trace() + self.assertEqual(ref_count, base_refs + 1) + + b = r.b + c = r.c + _, ref_count = r.trace() + self.assertEqual(ref_count, base_refs + 3) + +class TestImplicitFreeze(unittest.TestCase): + def test_implicit_freeze_func(self): + @freezable + def some_func(): + pass + r = Region() + + r.obj = some_func + self.assertFalse(is_frozen(r.obj)) + r.trace() + self.assertTrue(is_frozen(r.obj)) + + def test_implicit_freeze_type(self): + @freezable + class A: + pass + r = Region() + + r.obj = A + self.assertFalse(is_frozen(r.obj)) + r.trace() + self.assertTrue(is_frozen(r.obj)) + + def test_implicit_freeze_module(self): + import random; + r = Region() + + r.obj = random + self.assertFalse(is_frozen(r.obj)) + r.trace() + self.assertTrue(is_frozen(r.obj)) + + # Unimport module + sys.modules.pop("random", None) + sys.mut_modules.pop("random", None) + + def test_implicit_freeze_str(self): + r = Region() + + r.obj = "Ducks are cool" + r.trace() + self.assertTrue(is_frozen(r.obj)) + + def test_implicit_freeze_int(self): + r = Region() + + r.obj = 17 + r.trace() + self.assertTrue(is_frozen(r.obj)) diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index a0602f6bb5b60d..1d28de214535a7 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -291,8 +291,7 @@ get_reachable_proc(PyTypeObject *tp) static void gc_list_dissolve(PyGC_Head *list) { struct _gc_runtime_state* gc_state = get_gc_state(); - // Use `old[0]` here, we are setting the visited space to 0 in add_visited_set(). - gc_list_merge(list, &(gc_state->old[0].head)); + gc_list_merge(list, &(gc_state->young.head)); } typedef struct { @@ -310,7 +309,7 @@ typedef struct { PyObject *pending; } trace_state; -static void trace_state_destroy(trace_state* state, bool dissolve_gc) { +static void trace_state_destroy(trace_state* state) { if (state->visited) { _Py_hashtable_destroy(state->visited); state->visited = NULL; @@ -319,13 +318,9 @@ static void trace_state_destroy(trace_state* state, bool dissolve_gc) { Py_DECREF(state->pending); state->pending = NULL; } - - if (dissolve_gc) { - gc_list_dissolve(state->gc_list); - } } static int trace_state_init(trace_state* state, PyGC_Head *gc_list) { - assert(gc_list_is_empty(gc_list)); + assert(gc_list == NULL || gc_list_is_empty(gc_list)); state->visited = NULL; state->pending = NULL; @@ -349,7 +344,7 @@ static int trace_state_init(trace_state* state, PyGC_Head *gc_list) { return 0; error: - trace_state_destroy(state, false); + trace_state_destroy(state); return -1; } @@ -401,8 +396,8 @@ static int _move_obj(PyObject* obj, trace_state* state) { return -1; } - // This makes sure the object is removed from the local GC list. - if (PyObject_IS_GC(obj) && PyObject_GC_IsTracked(obj)) { + // This moves the object into the region list, if provided. + if (state->gc_list && PyObject_IS_GC(obj) && PyObject_GC_IsTracked(obj)) { // This flag may be set if the region is constructed as part of // a finalizer. If the flag remains set, for an object removed // from its GC list bad things can happen. @@ -441,15 +436,6 @@ static int _trace_once(PyObject* obj, trace_result* result, PyGC_Head *gc_list) trace(" - starting trace from %p", obj); int res = TRACE_RES_DONE; - // Make sure gc_list is valid - PyGC_Head local_gc_list; - bool dissolve_gc = false; - if (gc_list == NULL) { - gc_list_init(&local_gc_list); - gc_list = &local_gc_list; - dissolve_gc = true; - } - // init the trace state trace_state state; if (trace_state_init(&state, gc_list)) { @@ -475,12 +461,11 @@ static int _trace_once(PyObject* obj, trace_result* result, PyGC_Head *gc_list) goto finally; error: - dissolve_gc = true; res = TRACE_RES_ERR; finally: result->incoming_refs = state.external_rc; result->objs = _Py_hashtable_len(state.visited); - trace_state_destroy(&state, dissolve_gc); + trace_state_destroy(&state); return res; } @@ -494,8 +479,14 @@ static int trace_object(PyObject* obj, trace_result* result, PyGC_Head *gc_list) // Trace object int res = _trace_once(obj, result, gc_list); + + // Restart trace on demand if (res == TRACE_RES_RESTART) { trace("- restarting trace for %p", obj); + if (gc_list != NULL) { + gc_list_dissolve(gc_list); + assert(gc_list_is_empty(gc_list)); + } continue; } return res; @@ -530,6 +521,10 @@ TracingRegion_traverse(TracingRegionObject *self, visitproc visit, void *arg) { static int TracingRegion_clear(TracingRegionObject *self) { Py_CLEAR(self->dict); + // This is deallocating a closed region, we just dissolve it + if (!gc_list_is_empty(&self->gc_list)) { + gc_list_dissolve(&self->gc_list); + } return 0; } @@ -554,6 +549,51 @@ static PyObject* TracingRegion_trace(PyObject *op) { return t; } +/* This method traces the region and closes it if the caller has the only + * owning reference into the graph. The reference passed into this function + * needs to be borrowed. + * + * This function requires the GIL to be held. + * + * Returns -1 if an exception was raised. 0 if the region couldn't be closed + * and 1 if the region was closed. + */ +int _PyTracingRegion_Close(PyObject* op) { + TracingRegionObject *self = (TracingRegionObject*)op; + assert(gc_list_is_empty(&self->gc_list)); + + trace_result result; + if (trace_object(op, &result, &self->gc_list)) { + return -1; // propagate Python exception + } + + // Keep the region open, if the there are more incoming references + // besides the expected owning one + if (result.incoming_refs > 1) { + trace("- Failed to close region %p, there are %zd incoming references", self, result.incoming_refs); + gc_list_dissolve(&self->gc_list); + assert(gc_list_is_empty(&self->gc_list)); + return 1; + } + + trace("- Closed region %p", self); + assert(!gc_list_is_empty(&self->gc_list)); + return 0; +} + +/* This method opens the region by dissolving it and all objects into the + * local GC list. + * + * This function requires the GIL to be held. + */ +int _PyTracingRegion_Open(PyObject* op) { + TracingRegionObject *self = (TracingRegionObject*)op; + assert(!gc_list_is_empty(&self->gc_list)); + gc_list_dissolve(&self->gc_list); + assert(gc_list_is_empty(&self->gc_list)); + return 0; +} + static PyMethodDef TracingRegion_methods[] = { {"trace", _PyCFunction_CAST(TracingRegion_trace), METH_NOARGS, "This traces the region and returns the number of incoming references"}, @@ -581,4 +621,6 @@ PyTypeObject _PyTracingRegion_Type = { .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, }; - +// TODO: Weak-references pointing into the trace are not handled +// TODO: Weak-references part of the trace are not handled +// From f112d8bf430647b387f0cd7c09a0a46f5d9b139e Mon Sep 17 00:00:00 2001 From: xFrednet Date: Thu, 16 Apr 2026 16:11:24 +0200 Subject: [PATCH 5/8] Weakrefs again --- Objects/tracingregionobject.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index 1d28de214535a7..6033ac17b9c579 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -3,6 +3,7 @@ #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_descrobject.h" +#include "pycore_weakref.h" // #define REGION_TRACING @@ -370,13 +371,18 @@ static int _move_obj(PyObject* obj, trace_state* state) { state->src, obj); return TRACE_RES_ERR; case Py_MOVABLE_FREEZE: - // Freeze the object, this can invalidate our `external_rc`, we restart after this trace + // Freeze the object, this can invalidate our `external_rc`, + // we restart after this trace trace(" - freezing %p", obj); if (_PyImmutability_Freeze(obj)) { return TRACE_RES_ERR; } state->restart = true; + // Setting the gc_list to NULL will stop objects from being moved + // between GC lists. Just a small thing we can avoid. The next (full) + // trace will have this set again. + state->gc_list = NULL; return 0; default: assert(false); @@ -453,6 +459,9 @@ static int _trace_once(PyObject* obj, trace_result* result, PyGC_Head *gc_list) trace(" - traversing %p", item); traverseproc proc = get_reachable_proc(Py_TYPE(item)); SUCCEEDS(proc(item, (visitproc)_trace_visit, (void*)&state)); + + // Weak refs need special handling + assert(!PyWeakref_Check(item)); } if (state.restart) { @@ -495,6 +504,20 @@ static int trace_object(PyObject* obj, trace_result* result, PyGC_Head *gc_list) return TRACE_RES_DONE; } +static void detach_weak_refs(PyGC_Head *gc_list) { + PyGC_Head *current = GC_NEXT(gc_list); + while (current != gc_list) { + PyObject *item = _Py_FROM_GC(current); +#ifdef PY_DEBUG + Py_ssize_t weak_ctn = _PyWeakref_GetWeakrefCount(item); + if (weak_ctn) { + trace("- Clearing %zd weak references to %p", weak_ctn, item); + } +#endif + _PyWeakref_ClearWeakRefsNoCallbacks(item); + } +} + // ################################################################### // Region Object // ################################################################### @@ -576,6 +599,10 @@ int _PyTracingRegion_Close(PyObject* op) { return 1; } + // FIXME: This can be optimized, for example by inserting all objects + // with weak refs in the beginning. + detach_weak_refs(&self->gc_list); + trace("- Closed region %p", self); assert(!gc_list_is_empty(&self->gc_list)); return 0; @@ -621,6 +648,4 @@ PyTypeObject _PyTracingRegion_Type = { .tp_reachable = _PyObject_ReachableVisitTypeAndTraverse, }; -// TODO: Weak-references pointing into the trace are not handled // TODO: Weak-references part of the trace are not handled -// From 247f0cb0a77b5a5031616f3d3d46052b83526860 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Thu, 16 Apr 2026 17:13:04 +0200 Subject: [PATCH 6/8] Cowns are working? --- Include/internal/pycore_cown.h | 29 ++ Lib/immutable.py | 1 + Makefile.pre.in | 1 + Modules/_immutablemodule.c | 8 + Objects/cownobject.c | 554 +++++++++++++++++++++++++ Objects/tracingregionobject.c | 10 +- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 2 + PCbuild/pythoncore.vcxproj.filters | 8 + 10 files changed, 614 insertions(+), 3 deletions(-) create mode 100644 Include/internal/pycore_cown.h create mode 100644 Objects/cownobject.c diff --git a/Include/internal/pycore_cown.h b/Include/internal/pycore_cown.h new file mode 100644 index 00000000000000..0345690cc015ab --- /dev/null +++ b/Include/internal/pycore_cown.h @@ -0,0 +1,29 @@ +#ifndef Py_INTERNAL_COWN_H +#define Py_INTERNAL_COWN_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +#include "object.h" +#include "exports.h" + +typedef struct _PyCownObject _PyCownObject; +#define _PyCownObject_CAST(op) _Py_CAST(_PyCownObject*, op) + +PyAPI_DATA(PyTypeObject) _PyCown_Type; + +typedef uint64_t _PyCown_ipid_t; +typedef uint64_t _PyCown_thread_id_t; + +PyAPI_FUNC(_PyCown_ipid_t) _PyCown_ThisInterpreterId(void); +PyAPI_FUNC(_PyCown_thread_id_t) _PyCown_ThisThreadId(void); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_COWN_H */ \ No newline at end of file diff --git a/Lib/immutable.py b/Lib/immutable.py index 40cce9c93cbefb..167273bc31cdbd 100644 --- a/Lib/immutable.py +++ b/Lib/immutable.py @@ -22,6 +22,7 @@ InterpreterLocal = _c.InterpreterLocal SharedField = _c.SharedField TracingRegion = _c.TracingRegion +Cown = _c.Cown # FIXME(immutable): For the longest time we used the name `isfrozen` # without the underscore. This keeps the function name for now, but diff --git a/Makefile.pre.in b/Makefile.pre.in index 1b55ebe01a2a85..d8ae75ed97237e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -528,6 +528,7 @@ OBJECT_OBJS= \ Objects/classobject.o \ Objects/codeobject.o \ Objects/complexobject.o \ + Objects/cownobject.o \ Objects/descrobject.o \ Objects/enumobject.o \ Objects/exceptions.o \ diff --git a/Modules/_immutablemodule.c b/Modules/_immutablemodule.c index 7a237c6e1a7a8c..2c5a5ac665a510 100644 --- a/Modules/_immutablemodule.c +++ b/Modules/_immutablemodule.c @@ -8,6 +8,7 @@ #include "Python.h" #include +#include "pycore_cown.h" #include "pycore_object.h" #include "pycore_immutability.h" #include "pycore_critical_section.h" @@ -658,6 +659,13 @@ immutable_exec(PyObject *module) { return -1; } + if (PyModule_AddType(module, &_PyCown_Type) != 0) { + return -1; + } + if (_PyImmutability_SetFreezable((PyObject*)&_PyCown_Type, _Py_FREEZABLE_YES) < 0) { + return -1; + } + if (PyModule_AddIntConstant(module, "FREEZABLE_YES", _Py_FREEZABLE_YES) != 0) { return -1; diff --git a/Objects/cownobject.c b/Objects/cownobject.c new file mode 100644 index 00000000000000..5e91a2f6f8a3fb --- /dev/null +++ b/Objects/cownobject.c @@ -0,0 +1,554 @@ +#include "Python.h" +#include "pymacro.h" + +#include "pycore_cown.h" +#include "pycore_lock.h" +#include "pycore_time.h" // _PyTime_FromSeconds() + +/* Macro that jumps to error, if the expression `x` does not succeed. */ +#define SUCCEEDS(x) { do { int r = (x); if (r != 0) goto error; } while (0); } + +#define Region_Check(x) Py_IS_TYPE((x), &_PyTracingRegion_Type) + +// The interpreter id 0 is used. This value will be used to indicate that +// no interpreter owns the cown. +#define RELEASED_IPID ((_PyCown_ipid_t)0xff00ff00ff00ff00LL) +#define GC_IPID ((_PyCown_ipid_t)0xffff00ff00ff00ffLL) +#define NO_BLOCKING_TIMEOUT -1 +#define UNSET_THREAD_ID ((_PyCown_ipid_t)0xff00000000000000LL) + +typedef enum CownLockStatus { + COWN_ACQUIRE_ERROR = -1, + COWN_ACQUIRE_FAIL = 0, + COWN_ACQUIRE_SUCCESS = 1 +} CownLockStatus; + +struct _PyCownObject { + PyObject_HEAD + /* The id of the interpreter that currently owns this cown. + * + * This value may be read from and written to from different threads. + * Only use atomic operations to access this field. + */ + // FIXME(cowns): xFrednet: Make sure that an interpreter releases all + // cowns on destruction. + _PyCown_ipid_t owning_ip; + + /* The id of the thread that unlocked this cown. + * + * This is provided as additional information to users, it is not validated + * or used by this cown implementation. + */ + _PyCown_thread_id_t locking_thread; + + /* The value stored in the cown. This value may be immutable, another cown + * or a region object. + */ + PyObject* value; + + /* A lock used, mainly to support timeouts and queueing for locking. + * All other functions should use `owning_ip` to determine if they can + * access the data or not. + * + * Python's mutexes already implement queueing and timeouts in a good way. + * Later we can role our own, if we need but for not this is better. Note + * that the optional GIL release from the lock should not be used, as it + * doesn't seem to account for waiting threads from different interpreters. + * Therefore, we are responsible for releasing and acquireing the GIL. + */ + PyMutex lock; +}; + +static _PyCown_ipid_t cown_get_owner(_PyCownObject *obj) { + return _Py_atomic_load_uint64(&obj->owning_ip); +} + +#define BAIL_UNLESS_OWNED_BY(o, owned_by, result) \ + do {\ + _PyCown_ipid_t owning_ip = cown_get_owner(_PyCownObject_CAST(o)); \ + if (owning_ip != owned_by) { \ + PyErr_Format( \ + PyExc_RuntimeError, \ + "attempted to access a cown owned by %llu from %llu", \ + owning_ip, owned_by); \ + return result; \ + } \ + } while (0); +#define BAIL_UNLESS_OWNED(o, result) BAIL_UNLESS_OWNED_BY(o, _PyCown_ThisInterpreterId(), result) +#define BAIL_UNLESS_OWNED_NULL(o) BAIL_UNLESS_OWNED(o, NULL) + +static int cown_set_value_unchecked(_PyCownObject* self, PyObject* value) { + // Update the value + Py_XSETREF(self->value, Py_NewRef(value)); + + return 0; +} + +static int cown_set_value(_PyCownObject* self, PyObject* value) { + BAIL_UNLESS_OWNED(self, -1); + + // Bridge objects are allowed + if (Region_Check(value)) { + return cown_set_value_unchecked(self, value); + } + + // Immutable objects are allowed + if (_Py_IsImmutable(value)) { + return cown_set_value_unchecked(self, value); + } + + // Local objects are forbidden + PyErr_Format( + PyExc_RuntimeError, + "attempted to store a local mutable object in a cown.\n" + "Only regions, cown, and immutable objects are allowed"); + + return -1; +} + +/* Attempt to lock the cown. + * + * Timeout values: + * (-1) => Non-blocking locking + * (0) => Block with no timeout + * (n) => Blocking with timeout + */ +static int cown_lock(_PyCownObject* self, PyTime_t timeout, _PyCown_ipid_t locking_ip, bool has_gil) { + // A blocking time should only be set, if this call holds the GIL + assert(has_gil || timeout == NO_BLOCKING_TIMEOUT); + + // Try to lock the mutex directly, without releasing the GIL first + PyLockStatus r = _PyMutex_LockTimed(&self->lock, 0, _Py_LOCK_DONT_DETACH); + + // The cown is currently owned by something else. Release the GIL and + // wait for the timeout. + if (r != PY_LOCK_ACQUIRED && timeout != NO_BLOCKING_TIMEOUT) { + // Release the GIL + Py_BEGIN_ALLOW_THREADS; + + // Attempt to lock the mutex. This uses a PyMutex for the locking, + // timeout and signal handling. + r = _PyMutex_LockTimed( + &self->lock, + timeout, + _Py_LOCK_DONT_DETACH | _PY_LOCK_HANDLE_SIGNALS + ); + + // Acquire the GIL + Py_END_ALLOW_THREADS; + } + + // The lock was interrupted + if (r == PY_LOCK_INTR) { + return COWN_ACQUIRE_ERROR; + } + + // The lock acquisition failed + if (r == PY_LOCK_FAILURE) { + return COWN_ACQUIRE_FAIL; + } + + // Set the owning_ip to the current interpreter, thereby taking ownership + _PyCown_ipid_t released_value = RELEASED_IPID; + if (!_Py_atomic_compare_exchange_uint64( + &self->owning_ip, + &released_value, + locking_ip) + ) { + // Failed to set owning_ip, this should never happen and points + // to a deeper issue. + PyErr_Format( + PyExc_RuntimeError, + "[BUG] failed to set owner on a locked cown\n" + "Cown: %U", + self + ); + + _PyMutex_Unlock(&self->lock); + return COWN_ACQUIRE_ERROR; + } + + // Set the locking thread. + if (has_gil) { + self->locking_thread = _PyCown_ThisThreadId(); + } else { + self->locking_thread = UNSET_THREAD_ID; + } + + if (self->value && Region_Check(self->value)) { + _PyTracingRegion_Open(self->value); + } + + return COWN_ACQUIRE_SUCCESS; +} + +/* Returns the interpreter id used by cowns. + * + * The caller must hold the GIL. + */ +_PyCown_ipid_t _PyCown_ThisInterpreterId(void) { + _PyCown_ipid_t ip = PyInterpreterState_GetID(PyInterpreterState_Get()); + // This should never happen... if it does... we have a problem... + assert(ip != RELEASED_IPID); + return ip; +} + +/* Returns the thread id used by cowns. + * + * The caller must hold the GIL. + */ +_PyCown_thread_id_t _PyCown_ThisThreadId(void) { + _PyCown_thread_id_t id = PyThreadState_GetID(PyThreadState_Get()); + return id; +} + +static int PyCown_init(_PyCownObject *self, PyObject *args, PyObject *kwds) { + // See if we got a value as a keyword argument + static char *kwlist[] = {"value", NULL}; + PyObject *value = Py_None; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &value)) { + return -1; + } + + // Init the cown as being acquired by the current interpreter + _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + _Py_atomic_store_uint64(&self->owning_ip, RELEASED_IPID); + if (cown_lock(self, NO_BLOCKING_TIMEOUT, this_ip, true) != COWN_ACQUIRE_SUCCESS) { + PyErr_Format( + PyExc_RuntimeError, + "Newly created cown couldn't be acquired by interpreter %lld (this)", + this_ip); + return -1; + } + + // Set the cown value using the internal function for full validation + SUCCEEDS(cown_set_value(self, value)); + + // Freeze the cown to enable atomic reference counting for it. + PyObject_GC_UnTrack(self); + SUCCEEDS(_PyImmutability_Freeze(_PyObject_CAST(self))); + + return 0; +error: + return -1; +} + +static int PyCown_traverse(_PyCownObject *self, visitproc _ignore1, void* _ignore2) { + // tp_traverse should never be called on cowns since they're not + // tracked by the GC or in any other GC list. The cown type + // still defines `tp_traverse` to ensure that this is never + // accidentally called. Later we may want to simple remove it + // from the type. + assert(false); + return -1; +} + +static int PyCown_reachable(_PyCownObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); + + // The value is explicitly not visited. Freezing or moving cowns should + // not propagate to the value. + // Py_VISIT(self->value); + + return 0; +} + +static int PyCown_clear(_PyCownObject *self) { + cown_set_value_unchecked(self, Py_None); + Py_CLEAR(self->value); + return 0; +} + +static void PyCown_dealloc(_PyCownObject *self) { + PyObject_GC_UnTrack(self); + PyCown_clear(self); + PyObject_GC_Del(self); +} + +static int +lock_acquire_parse_args(PyObject *args, PyObject *kwds, + PyTime_t *timeout) +{ + // Taken from `Modules/_threadmodule.c` + + char *kwlist[] = {"blocking", "timeout", NULL}; + int blocking = 1; + PyObject *timeout_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pO:acquire", kwlist, + &blocking, &timeout_obj)) + return -1; + + const PyTime_t unset_timeout = _PyTime_FromSeconds(NO_BLOCKING_TIMEOUT); + *timeout = unset_timeout; + + if (timeout_obj + && _PyTime_FromSecondsObject(timeout, + timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) + return -1; + + if (!blocking && *timeout != unset_timeout ) { + PyErr_SetString(PyExc_ValueError, + "can't specify a timeout for a non-blocking call"); + return -1; + } + if (*timeout < 0 && *timeout != unset_timeout) { + PyErr_SetString(PyExc_ValueError, + "timeout value must be a non-negative number"); + return -1; + } + if (!blocking) + *timeout = 0; + else if (*timeout != unset_timeout) { + PyTime_t microseconds; + + microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_TIMEOUT); + if (microseconds > PY_TIMEOUT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "timeout value is too large"); + return -1; + } + } + return 0; +} + +static PyObject * +CownObject_acquire(_PyCownObject *self, PyObject *args, PyObject *kwds) +{ + // Parse the arguments + PyTime_t timeout; + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { + return NULL; + } + + // Attempt to lock the cown + _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + int res = cown_lock(self, timeout, this_ip, true); + if (res == COWN_ACQUIRE_ERROR) { + return NULL; + } + + // Return the result + return PyBool_FromLong(res == COWN_ACQUIRE_SUCCESS); +} + +PyDoc_STRVAR(CownObject_acquire_doc, +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ +\n\ +Attempts to acquires the cown. With default arguments this will block\n\ +until the cown can be aquired, even when acquire is called from the same\n\ +interpreter. The return indicates if the cown was\n\ +was acquired. The blocking operation is interruptible."); + +static int cown_release_unchecked(_PyCownObject* self, _PyCown_ipid_t unlocking_ip) { + // Set owning_ip to indicate the released state + if (!_Py_atomic_compare_exchange_uint64(&self->owning_ip, &unlocking_ip, RELEASED_IPID)) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %lld (this) attempted to release a cown owned by someone else\n" + "Cown: %U", + unlocking_ip, self); + return -1; + } + + // Unlocking should always succeed + int res = _PyMutex_TryUnlock(&self->lock); + assert(res == 0); + (void)res; + + return 0; +} + +/* Checks that the cown is not released, and that the owner is as the current interpreter. */ +static int cown_check_owner_before_release(_PyCownObject *self, _PyCown_ipid_t unlocking_ip) { + _PyCown_ipid_t owning_ip = cown_get_owner(self); + if (owning_ip == RELEASED_IPID) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %lld attempted to release/switch a released cown", + unlocking_ip + ); + return -1; + } + if (owning_ip != unlocking_ip) { + PyErr_Format( + PyExc_RuntimeError, + "interpreter %lld attempted to release/switch a cown owned by %lld", + unlocking_ip, owning_ip + ); + return -1; + } + return 0; +} + +/* Try closing the region by cleaning it. + * Returns: + * (-1) If an error occurred while trying to clean the region. + * (0) If the region is still open after this call. + * (1) If the region is closed after this call. + */ +static int cown_try_closing_region(_PyCownObject *self) { + assert(Region_Check(self->value)); + + return _PyTracingRegion_Close(self->value); +} + +static int cown_release(_PyCownObject *self, _PyCown_ipid_t unlocking_ip) { + if (cown_check_owner_before_release(self, unlocking_ip) < 0) { + return -1; + } + + if (_Py_IsImmutable(self->value)) { + // Can be released without any restrictions + return cown_release_unchecked(self, unlocking_ip); + } + assert(Region_Check(self->value)); + + int cleaning_res = cown_try_closing_region(self); + if (cleaning_res < 0) { + return -1; + } + if (cleaning_res == 0) { + PyErr_Format( + PyExc_RuntimeError, + "the cown can't be released, since the contained region is still open"); + return -1; + } + // Region is closed, safe to release + return cown_release_unchecked(self, unlocking_ip); +} + +static PyObject* CownObject_release(_PyCownObject *self, PyObject *ignored) { + _PyCown_ipid_t this_ip = _PyCown_ThisInterpreterId(); + if (cown_release(self, this_ip) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(CownObject_release_doc, +"release($self, /)\n\ +--\n\ +\n\ +Release the cown, allowing another interpreter that is blocked waiting for\n\ +the cown to acquire the cown. The cown must be in the locked state\n\ +and must be unlocked from the owning interpreter. It may be unlocked \n\ +by any thread on the owning interpreter."); + +static PyObject * +CownObject_locked(_PyCownObject *op, PyObject *Py_UNUSED(dummy)) +{ + return PyBool_FromLong(cown_get_owner(op) != RELEASED_IPID); +} + +PyDoc_STRVAR(CownObject_locked_doc, +"locked($self, /)\n\ +--\n\ +\n\ +Return whether the cown currently released or aquired. \n\ +Use `owned()` to check if the cown is aquired by the current interpreter."); + +static PyObject * +CownObject_owned(_PyCownObject *op, PyObject *Py_UNUSED(dummy)) +{ + return PyBool_FromLong(cown_get_owner(op) == _PyCown_ThisInterpreterId()); +} + +PyDoc_STRVAR(CownObject_owned_doc, +"owned($self, /)\n\ +--\n\ +\n\ +Return true if the cown is currently aquired by this interpreter, false otherwise."); + +static PyObject * +CownObject_owned_by_thread(_PyCownObject *op, PyObject *Py_UNUSED(dummy)) +{ + if (cown_get_owner(op) != _PyCown_ThisInterpreterId()) { + Py_RETURN_FALSE; + } + + return PyBool_FromLong(op->locking_thread == _PyCown_ThisThreadId()); +} + +PyDoc_STRVAR(CownObject_owned_by_thread_doc, +"owned($self, /)\n\ +--\n\ +\n\ +Return true if the cown is currently aquired by this interpreter and was \n\ +locked by the current thread, false otherwise. \n\ +Ownership on the thread level is not enforced, any thread on the owning\n\ +interpreter can access and release the cown. This is information is only\n\ +provided to give more control for those who seek it."); + + +// Define the CownType with methods +static PyMethodDef PyCown_methods[] = { + {"acquire", _PyCFunction_CAST(CownObject_acquire), METH_VARARGS | METH_KEYWORDS, CownObject_acquire_doc}, + {"release", _PyCFunction_CAST(CownObject_release), METH_NOARGS, CownObject_release_doc}, + {"locked", _PyCFunction_CAST(CownObject_locked), METH_NOARGS, CownObject_locked_doc}, + {"owned", _PyCFunction_CAST(CownObject_owned), METH_NOARGS, CownObject_owned_doc}, + {"owned_by_thread", _PyCFunction_CAST(CownObject_owned_by_thread), METH_NOARGS, CownObject_owned_by_thread_doc}, + {NULL} // Sentinel +}; + +static PyObject *CownObject_get_value(_PyCownObject *self, void *closure) { + BAIL_UNLESS_OWNED_NULL(self); + + return Py_NewRef(self->value); +} + +static int CownObject_set_value(_PyCownObject *self, PyObject *value, void *closure) { + BAIL_UNLESS_OWNED(self, -1); + + return cown_set_value(self, value); +} + +static PyGetSetDef PyCownObject_getset[] = { + {"value", (getter)CownObject_get_value, (setter)CownObject_set_value, + "", NULL}, + {NULL, NULL, NULL, NULL, NULL} +}; + +static PyObject *PyCown_repr(_PyCownObject *self) { + _PyCown_ipid_t owner = cown_get_owner(self); + // On this interpreter we can access the cown and content + // safely since we hold the GIL + if (owner == _PyCown_ThisInterpreterId()) { + return PyUnicode_FromFormat( + "Cown(interpreter=%llu (this), value=%S)", + owner, + PyObject_Repr(self->value) + ); + } + + // The cown is released and can be acquired + if (owner == RELEASED_IPID) { + return PyUnicode_FromFormat( + "Cown(interpreter=None, status=Released)" + ); + } + + // The cown is owned by a different interpreter + return PyUnicode_FromFormat( + "Cown(interpreter=%llu (other))", + owner + ); +} + +PyTypeObject _PyCown_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "Cown", + .tp_basicsize = sizeof(_PyCownObject), + .tp_dealloc = (destructor)PyCown_dealloc, + .tp_repr = (reprfunc)PyCown_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .tp_traverse = (traverseproc)PyCown_traverse, + .tp_reachable = (traverseproc)PyCown_reachable, + .tp_clear = (inquiry)PyCown_clear, + .tp_methods = PyCown_methods, + .tp_getset = PyCownObject_getset, + .tp_init = (initproc)PyCown_init, + .tp_new = PyType_GenericNew, +}; + diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index 6033ac17b9c579..ddc16e1a1fb01e 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -514,7 +514,11 @@ static void detach_weak_refs(PyGC_Head *gc_list) { trace("- Clearing %zd weak references to %p", weak_ctn, item); } #endif - _PyWeakref_ClearWeakRefsNoCallbacks(item); + if (_PyType_SUPPORTS_WEAKREFS(Py_TYPE(item))) { + _PyWeakref_ClearWeakRefsNoCallbacks(item); + } + + current = GC_NEXT(current); } } @@ -596,7 +600,7 @@ int _PyTracingRegion_Close(PyObject* op) { trace("- Failed to close region %p, there are %zd incoming references", self, result.incoming_refs); gc_list_dissolve(&self->gc_list); assert(gc_list_is_empty(&self->gc_list)); - return 1; + return 0; } // FIXME: This can be optimized, for example by inserting all objects @@ -605,7 +609,7 @@ int _PyTracingRegion_Close(PyObject* op) { trace("- Closed region %p", self); assert(!gc_list_is_empty(&self->gc_list)); - return 0; + return 1; } /* This method opens the region by dissolving it and all objects into the diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index c2514ce954c902..c19b7efdd18153 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -134,6 +134,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index ebb3c7a469b443..0a33235d9dc855 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -106,6 +106,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b61f2669d4974d..32d5877122e494 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -238,6 +238,7 @@ + @@ -532,6 +533,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 61fe05d775a7b5..0106e8290c20fa 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -618,6 +618,9 @@ Include\internal + + Include\internal + Include\internal @@ -697,6 +700,8 @@ Include\internal + Include\internal + Include\internal @@ -1207,6 +1212,9 @@ Objects + + Objects + Objects From 6271692724d30db6f843de9a23c5319950f6bef1 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 21 Apr 2026 11:00:52 +0200 Subject: [PATCH 7/8] IDK --- Objects/tracingregionobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index ddc16e1a1fb01e..a9cda30698f776 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -292,7 +292,8 @@ get_reachable_proc(PyTypeObject *tp) static void gc_list_dissolve(PyGC_Head *list) { struct _gc_runtime_state* gc_state = get_gc_state(); - gc_list_merge(list, &(gc_state->young.head)); + //gc_list_merge(list, &(gc_state->young.head)); + gc_list_merge(list, &(gc_state->old[0].head)); } typedef struct { From bf4b7b0391b4a46cf5a1054b2f2ec9960374a17e Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 28 Apr 2026 09:52:38 +0200 Subject: [PATCH 8/8] Memory fun --- Objects/tracingregionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/tracingregionobject.c b/Objects/tracingregionobject.c index a9cda30698f776..50053331634d59 100644 --- a/Objects/tracingregionobject.c +++ b/Objects/tracingregionobject.c @@ -548,11 +548,11 @@ TracingRegion_traverse(TracingRegionObject *self, visitproc visit, void *arg) { static int TracingRegion_clear(TracingRegionObject *self) { - Py_CLEAR(self->dict); // This is deallocating a closed region, we just dissolve it if (!gc_list_is_empty(&self->gc_list)) { gc_list_dissolve(&self->gc_list); } + Py_CLEAR(self->dict); return 0; }