defaultdict: remove __missing__, add __getitem__

Author: Changaco

Doc/library/collections.rst

@@ -740,43 +740,38 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``,
     arguments.
 
 
-    :class:`defaultdict` objects support the following method in addition to the
-    standard :class:`dict` operations:
+    :class:`defaultdict` overrides the following method:
 
-    .. method:: __missing__(key, /)
+    .. method:: __getitem__(key, /)
 
-        If the :attr:`default_factory` attribute is ``None``, this raises a
-        :exc:`KeyError` exception with the *key* as argument.
+        Return ``self[key]``. If the item doesn't exist, it is automatically created.
+        The value is generated by calling either the :meth:`~object.__missing__` method
+        (if it exists) or the :attr:`default_factory` attribute (if it isn't None). If
+        neither can be called, a :exc:`KeyError` is raised.
 
-        If :attr:`default_factory` is not ``None``, it is called without arguments
-        to provide a default value for the given *key*, this value is inserted in
-        the dictionary for the *key*, and returned.
-
-        If calling :attr:`default_factory` raises an exception this exception is
-        propagated unchanged.
-
-        This method is called by the :meth:`~object.__getitem__` method of the
-        :class:`dict` class when the requested key is not found; whatever it
-        returns or raises is then returned or raised by :meth:`~object.__getitem__`.
-
-        Note that :meth:`__missing__` is *not* called for any operations besides
-        :meth:`~object.__getitem__`. This means that :meth:`~dict.get` will, like
-        normal dictionaries, return ``None`` as a default rather than using
-        :attr:`default_factory`.
+        When :term:`free threading` is enabled, the defaultdict is locked while the
+        key is being looked up and the default value is being generated.
 
 
     :class:`defaultdict` objects support the following instance variable:
 
-
     .. attribute:: default_factory
 
-        This attribute is used by the :meth:`~defaultdict.__missing__` method;
-        it is initialized from the first argument to the constructor, if present,
-        or to ``None``, if absent.
+        This attribute is called by the :meth:`defaultdict.__getitem__` method
+        if the requested key isn't in the dictionary. It must be either a
+        callable that takes no arguments, or :const:`None`.
+
 
     .. versionchanged:: 3.9
-       Added merge (``|``) and update (``|=``) operators, specified in
-       :pep:`584`.
+        Added merge (``|``) and update (``|=``) operators, specified in
+        :pep:`584`.
+
+    .. versionchanged:: 3.15
+        The built-in :meth:`defaultdict.__missing__` method no longer exists. A
+        custom :meth:`~object.__missing__` method should no longer insert the
+        generated value into the dictionary, as this is done by the new
+        :meth:`__getitem__` method. defaultdict is now safe to use with
+        :term:`free threading`.
 
 
 :class:`defaultdict` Examples

Include/internal/pycore_dict.h

@@ -123,6 +123,8 @@ PyAPI_FUNC(Py_ssize_t) _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_
 extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr);
 extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr);
 
+extern void _Py_dict_unhashable_type(PyObject *op, PyObject *key);
+
 extern int _PyDict_GetMethodStackRef(PyDictObject *dict, PyObject *name, _PyStackRef *method);
 
 // Exported for external JIT support

Lib/importlib/metadata/__init__.py

@@ -22,6 +22,7 @@
 import sys
 import textwrap
 import types
+from collections import defaultdict
 from collections.abc import Iterable, Mapping
 from contextlib import suppress
 from importlib import import_module
@@ -30,7 +31,7 @@
 from typing import Any
 
 from . import _meta
-from ._collections import FreezableDefaultDict, Pair
+from ._collections import Pair
 from ._context import ExceptionTrap
 from ._functools import method_cache, noop, pass_none, passthrough
 from ._itertools import always_iterable, bucket, unique_everseen
@@ -889,8 +890,8 @@ def __init__(self, path: FastPath):
 
         base = os.path.basename(path.root).lower()
         base_is_egg = base.endswith(".egg")
-        self.infos = FreezableDefaultDict(list)
-        self.eggs = FreezableDefaultDict(list)
+        self.infos = defaultdict(list)
+        self.eggs = defaultdict(list)
 
         for child in path.children():
             low = child.lower()
@@ -904,20 +905,17 @@ def __init__(self, path: FastPath):
                 legacy_normalized = Prepared.legacy_normalize(name)
                 self.eggs[legacy_normalized].append(path.joinpath(child))
 
-        self.infos.freeze()
-        self.eggs.freeze()
-
     def search(self, prepared: Prepared):
         """
         Yield all infos and eggs matching the Prepared query.
         """
         infos = (
-            self.infos[prepared.normalized]
+            self.infos.get(prepared.normalized, ())
             if prepared
             else itertools.chain.from_iterable(self.infos.values())
         )
         eggs = (
-            self.eggs[prepared.legacy_normalized]
+            self.eggs.get(prepared.legacy_normalized, ())
             if prepared
             else itertools.chain.from_iterable(self.eggs.values())
         )

Lib/importlib/metadata/_collections.py

@@ -1,30 +1,6 @@
-import collections
 import typing
 
 
-# from jaraco.collections 3.3
-class FreezableDefaultDict(collections.defaultdict):
-    """
-    Often it is desirable to prevent the mutation of
-    a default dict after its initial construction, such
-    as to prevent mutation during iteration.
-
-    >>> dd = FreezableDefaultDict(list)
-    >>> dd[0].append('1')
-    >>> dd.freeze()
-    >>> dd[1]
-    []
-    >>> len(dd)
-    1
-    """
-
-    def __missing__(self, key):
-        return getattr(self, '_frozen', super().__missing__)(key)
-
-    def freeze(self):
-        self._frozen = lambda key: self.default_factory()
-
-
 class Pair(typing.NamedTuple):
     name: str
     value: str

Lib/pydoc_data/topics.py

@@ -14,6 +14,7 @@ class TestDefaultDict(unittest.TestCase):
     def test_basic(self):
         d1 = defaultdict()
         self.assertEqual(d1.default_factory, None)
+        self.assertRaises(KeyError, d1.__getitem__, 42)
         d1.default_factory = list
         d1[12].append(42)
         self.assertEqual(d1, {12: [42]})
@@ -48,14 +49,15 @@ def test_basic(self):
         self.assertRaises(TypeError, defaultdict, 1)
 
     def test_missing(self):
-        d1 = defaultdict()
-        self.assertRaises(KeyError, d1.__missing__, 42)
-        d1.default_factory = list
-        v1 = d1.__missing__(42)
-        self.assertEqual(v1, [])
-        v2 = d1.__missing__(42)
-        self.assertEqual(v2, [])
-        self.assertIsNot(v2, v1)
+        # Check that __missing__ is called when it exists
+        class A(defaultdict):
+            def __missing__(self, key):
+                return []
+        d1 = A()
+        self.assertEqual(d1.__missing__(1), [])
+        # Check that default_factory isn't called when __missing__ exists
+        d1.default_factory = dict
+        self.assertEqual(d1.__missing__(2), [])
 
     def test_repr(self):
         d1 = defaultdict()

Lib/test/test_defaultdict.py

@@ -2222,31 +2222,56 @@ typedef struct {
 
 static PyType_Spec defdict_spec;
 
-PyDoc_STRVAR(defdict_missing_doc,
-"__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n\
-  if self.default_factory is None: raise KeyError((key,))\n\
-  self[key] = value = self.default_factory()\n\
-  return value\n\
+PyDoc_STRVAR(defdict_getitem_doc,
+"__getitem__($self, key, /)\n--\n\n\
+Return self[key]. If the item doesn't exist, it is automatically created.\n\
+The value is generated by calling either the __missing__ method (if it exists)\n\
+or the default_factory attribute (if it isn't None). If neither can be called,\n\
+a KeyError is raised.\
 ");
 
 static PyObject *
-defdict_missing(PyObject *op, PyObject *key)
+defdict_subscript(PyObject *op, PyObject *key)
 {
-    defdictobject *dd = defdictobject_CAST(op);
-    PyObject *factory = dd->default_factory;
+    PyDictObject *mp = (PyDictObject *)op;
+    Py_ssize_t ix;
+    Py_hash_t hash;
     PyObject *value;
-    if (factory == NULL || factory == Py_None) {
-        /* XXX Call dict.__missing__(key) */
-        _PyErr_SetKeyError(key);
+
+    hash = _PyObject_HashFast(key);
+    if (hash == -1) {
+        _Py_dict_unhashable_type(op, key);
         return NULL;
     }
-    value = _PyObject_CallNoArgs(factory);
-    if (value == NULL)
-        return value;
-    if (PyObject_SetItem(op, key, value) < 0) {
-        Py_DECREF(value);
-        return NULL;
+    Py_BEGIN_CRITICAL_SECTION(op);
+    ix = _Py_dict_lookup(mp, key, hash, &value);
+    if (value != NULL) {
+        Py_INCREF(value);
+    } else if (ix != DKIX_ERROR) {
+        /* Try to call self.__missing__(key) */
+        PyObject *missing;
+        int ret = PyObject_GetOptionalAttr(op, &_Py_ID(__missing__), &missing);
+        if (ret == 1) {
+            value = PyObject_CallOneArg(missing, key);
+            Py_DECREF(missing);
+        } else if (ret == 0) {
+            /* Try to call self.default_factory() */
+            PyObject *factory = defdictobject_CAST(op)->default_factory;
+            if (factory != NULL && factory != Py_None) {
+                value = _PyObject_CallNoArgs(factory);
+            } else {
+                _PyErr_SetKeyError(key);
+            }
+        }
+        /* Try to insert the new value in the dict */
+        if (value != NULL) {
+            ret = _PyDict_SetItem_KnownHash_LockHeld(mp, Py_NewRef(key),
+                                                     Py_NewRef(value), hash);
+            if (ret < 0)
+                value = NULL;
+        }
     }
+    Py_END_CRITICAL_SECTION();
     return value;
 }
 
@@ -2327,8 +2352,8 @@ defdict_reduce(PyObject *op, PyObject *Py_UNUSED(dummy))
 }
 
 static PyMethodDef defdict_methods[] = {
-    {"__missing__", defdict_missing, METH_O,
-     defdict_missing_doc},
+    {"__getitem__", defdict_subscript, METH_O|METH_COEXIST,
+     defdict_getitem_doc},
     {"copy", defdict_copy, METH_NOARGS,
      defdict_copy_doc},
     {"__copy__", defdict_copy, METH_NOARGS,
@@ -2343,7 +2368,7 @@ static PyMethodDef defdict_methods[] = {
 static PyMemberDef defdict_members[] = {
     {"default_factory", _Py_T_OBJECT,
      offsetof(defdictobject, default_factory), 0,
-     PyDoc_STR("Factory for default value called by __missing__().")},
+     PyDoc_STR("Factory for default value, called by __getitem__().")},
     {NULL}
 };
 
@@ -2507,6 +2532,7 @@ static PyType_Slot defdict_slots[] = {
     {Py_tp_init, defdict_init},
     {Py_tp_alloc, PyType_GenericAlloc},
     {Py_tp_free, PyObject_GC_Del},
+    {Py_mp_subscript, defdict_subscript},
     {0, NULL},
 };