[Python-checkins] gh-81057: Move the Extension Modules Cache to _PyRuntimeState (gh-99355)

Fri Nov 11 16:16:57 EST 2022

https://github.com/python/cpython/commit/dd36b71fa6164ebba5d94bb4a24eac43b1c54906
commit: dd36b71fa6164ebba5d94bb4a24eac43b1c54906
branch: main
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: ericsnowcurrently <ericsnowcurrently at gmail.com>
date: 2022年11月11日T14:16:28-07:00
summary:
gh-81057: Move the Extension Modules Cache to _PyRuntimeState (gh-99355)
We also move the closely related max_module_number and add comments documenting the group of struct members.
https://github.com/python/cpython/issues/81057
files:
M Include/internal/pycore_import.h
M Include/internal/pycore_interp.h
M Include/internal/pycore_runtime.h
M Include/moduleobject.h
M Objects/moduleobject.c
M Python/import.c
M Tools/c-analyzer/cpython/globals-to-fix.tsv

diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h
index aee1f66a3ea1..7f1240f7c9db 100644
--- a/Include/internal/pycore_import.h
+++ b/Include/internal/pycore_import.h
@@ -5,6 +5,23 @@
 extern "C" {
 #endif
 
+
+struct _import_runtime_state {
+ /* The most recent value assigned to a PyModuleDef.m_base.m_index.
+ This is incremented each time PyModuleDef_Init() is called,
+ which is just about every time an extension module is imported.
+ See PyInterpreterState.modules_by_index for more info. */
+ Py_ssize_t last_module_index;
+ /* A dict mapping (filename, name) to PyModuleDef for modules.
+ Only legacy (single-phase init) extension modules are added
+ and only if they support multiple initialization (m_size >- 0)
+ or are imported in the main interpreter.
+ This is initialized lazily in _PyImport_FixupExtensionObject().
+ Modules are added there and looked up in _imp.find_extension(). */
+ PyObject *extensions;
+};
+
+
 #ifdef HAVE_FORK
 extern PyStatus _PyImport_ReInitLock(void);
 #endif
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index ae2a3d3b13cf..068b0a700af5 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -123,6 +123,25 @@ struct _is {
 
 // sys.modules dictionary
 PyObject *modules;
+ /* This is the list of module objects for all legacy (single-phase init)
+ extension modules ever loaded in this process (i.e. imported
+ in this interpreter or in any other). Py_None stands in for
+ modules that haven't actually been imported in this interpreter.
+
+ A module's index (PyModuleDef.m_base.m_index) is used to look up
+ the corresponding module object for this interpreter, if any.
+ (See PyState_FindModule().) When any extension module
+ is initialized during import, its moduledef gets initialized by
+ PyModuleDef_Init(), and the first time that happens for each
+ PyModuleDef, its index gets set to the current value of
+ a global counter (see _PyRuntimeState.imports.last_module_index).
+ The entry for that index in this interpreter remains unset until
+ the module is actually imported here. (Py_None is used as
+ a placeholder.) Note that multi-phase init modules always get
+ an index for which there will never be a module set.
+
+ This is initialized lazily in _PyState_AddModule(), which is also
+ where modules get added. */
 PyObject *modules_by_index;
 // Dictionary of the sys module
 PyObject *sysdict;
diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h
index d1fbc09f1ea2..df35e34291af 100644
--- a/Include/internal/pycore_runtime.h
+++ b/Include/internal/pycore_runtime.h
@@ -11,6 +11,7 @@ extern "C" {
 #include "pycore_atomic.h" /* _Py_atomic_address */
 #include "pycore_gil.h" // struct _gil_runtime_state
 #include "pycore_global_objects.h" // struct _Py_global_objects
+#include "pycore_import.h" // struct _import_runtime_state
 #include "pycore_interp.h" // PyInterpreterState
 #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids
 
@@ -115,6 +116,7 @@ typedef struct pyruntimestate {
 void (*exitfuncs[NEXITFUNCS])(void);
 int nexitfuncs;
 
+ struct _import_runtime_state imports;
 struct _ceval_runtime_state ceval;
 struct _gilstate_runtime_state gilstate;
 struct _getargs_runtime_state getargs;
diff --git a/Include/moduleobject.h b/Include/moduleobject.h
index fbb2c5ae7944..555564ec73b4 100644
--- a/Include/moduleobject.h
+++ b/Include/moduleobject.h
@@ -43,8 +43,22 @@ PyAPI_DATA(PyTypeObject) PyModuleDef_Type;
 
 typedef struct PyModuleDef_Base {
 PyObject_HEAD
+ /* The function used to re-initialize the module.
+ This is only set for legacy (single-phase init) extension modules
+ and only used for those that support multiple initializations
+ (m_size >= 0).
+ It is set by _PyImport_LoadDynamicModuleWithSpec()
+ and _imp.create_builtin(). */
 PyObject* (*m_init)(void);
+ /* The module's index into its interpreter's modules_by_index cache.
+ This is set for all extension modules but only used for legacy ones.
+ (See PyInterpreterState.modules_by_index for more info.)
+ It is set by PyModuleDef_Init(). */
 Py_ssize_t m_index;
+ /* A copy of the module's __dict__ after the first time it was loaded.
+ This is only set/used for legacy modules that do not support
+ multiple initializations.
+ It is set by _PyImport_FixupExtensionObject(). */
 PyObject* m_copy;
 } PyModuleDef_Base;
 
diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c
index a1d09a6e4604..4a423a719e90 100644
--- a/Objects/moduleobject.c
+++ b/Objects/moduleobject.c
@@ -9,7 +9,6 @@
 #include "pycore_moduleobject.h" // _PyModule_GetDef()
 #include "structmember.h" // PyMemberDef
 
-static Py_ssize_t max_module_number;
 
 static PyMemberDef module_members[] = {
 {"__dict__", T_OBJECT, offsetof(PyModuleObject, md_dict), READONLY},
@@ -43,10 +42,10 @@ PyModuleDef_Init(PyModuleDef* def)
 {
 assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY);
 if (def->m_base.m_index == 0) {
- max_module_number++;
+ _PyRuntime.imports.last_module_index++;
 Py_SET_REFCNT(def, 1);
 Py_SET_TYPE(def, &PyModuleDef_Type);
- def->m_base.m_index = max_module_number;
+ def->m_base.m_index = _PyRuntime.imports.last_module_index;
 }
 return (PyObject*)def;
 }
diff --git a/Python/import.c b/Python/import.c
index 2fd2d1b6b89d..d16161381a97 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -27,9 +27,6 @@ extern "C" {
 /* Forward references */
 static PyObject *import_add_module(PyThreadState *tstate, PyObject *name);
 
-/* See _PyImport_FixupExtensionObject() below */
-static PyObject *extensions = NULL;
-
 /* This table is defined in config.c: */
 extern struct _inittab _PyImport_Inittab[];
 
@@ -221,10 +218,12 @@ _imp_release_lock_impl(PyObject *module)
 Py_RETURN_NONE;
 }
 
+static inline void _extensions_cache_clear(void);
+
 void
 _PyImport_Fini(void)
 {
- Py_CLEAR(extensions);
+ _extensions_cache_clear();
 if (import_lock != NULL) {
 PyThread_free_lock(import_lock);
 import_lock = NULL;
@@ -398,6 +397,51 @@ PyImport_GetMagicTag(void)
 dictionary, to avoid loading shared libraries twice.
 */
 
+static PyModuleDef *
+_extensions_cache_get(PyObject *filename, PyObject *name)
+{
+ PyObject *extensions = _PyRuntime.imports.extensions;
+ if (extensions == NULL) {
+ return NULL;
+ }
+ PyObject *key = PyTuple_Pack(2, filename, name);
+ if (key == NULL) {
+ return NULL;
+ }
+ PyModuleDef *def = (PyModuleDef *)PyDict_GetItemWithError(extensions, key);
+ Py_DECREF(key);
+ return def;
+}
+
+static int
+_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def)
+{
+ PyObject *extensions = _PyRuntime.imports.extensions;
+ if (extensions == NULL) {
+ extensions = PyDict_New();
+ if (extensions == NULL) {
+ return -1;
+ }
+ _PyRuntime.imports.extensions = extensions;
+ }
+ PyObject *key = PyTuple_Pack(2, filename, name);
+ if (key == NULL) {
+ return -1;
+ }
+ int res = PyDict_SetItem(extensions, key, (PyObject *)def);
+ Py_DECREF(key);
+ if (res < 0) {
+ return -1;
+ }
+ return 0;
+}
+
+static void
+_extensions_cache_clear(void)
+{
+ Py_CLEAR(_PyRuntime.imports.extensions);
+}
+
 int
 _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
 PyObject *filename, PyObject *modules)
@@ -442,20 +486,7 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
 }
 }
 
- if (extensions == NULL) {
- extensions = PyDict_New();
- if (extensions == NULL) {
- return -1;
- }
- }
-
- PyObject *key = PyTuple_Pack(2, filename, name);
- if (key == NULL) {
- return -1;
- }
- int res = PyDict_SetItem(extensions, key, (PyObject *)def);
- Py_DECREF(key);
- if (res < 0) {
+ if (_extensions_cache_set(filename, name, def) < 0) {
 return -1;
 }
 }
@@ -480,16 +511,7 @@ static PyObject *
 import_find_extension(PyThreadState *tstate, PyObject *name,
 PyObject *filename)
 {
- if (extensions == NULL) {
- return NULL;
- }
-
- PyObject *key = PyTuple_Pack(2, filename, name);
- if (key == NULL) {
- return NULL;
- }
- PyModuleDef* def = (PyModuleDef *)PyDict_GetItemWithError(extensions, key);
- Py_DECREF(key);
+ PyModuleDef *def = _extensions_cache_get(filename, name);
 if (def == NULL) {
 return NULL;
 }
diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index 4cd29a8a0b0c..bb05a2c469bd 100644
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -317,7 +317,6 @@ Python/hamt.c	-	_empty_hamt	-
 
 # state
 Objects/typeobject.c	resolve_slotdups	pname	-
-Python/import.c	-	extensions	-
 
 
 ##################################
@@ -449,7 +448,6 @@ Python/getargs.c	-	static_arg_parsers	-
 Objects/dictobject.c	-	_pydict_global_version	-
 Objects/dictobject.c	-	next_dict_keys_version	-
 Objects/funcobject.c	-	next_func_version	-
-Objects/moduleobject.c	-	max_module_number	-
 Objects/object.c	-	_Py_RefTotal	-
 Python/perf_trampoline.c	-	perf_status	-
 Python/perf_trampoline.c	-	extra_code_index	-
