Lazy Imports

Summary:
This adds Lazy Imports to CPython 3.12

 ---

The following diffs from Cinder 3.10 needed to be ported to 3.12, on top of the specification in PEP-690:

+ D44148110 - Simplify lazy objects (always single level, less attributes)
+ D44220896 - Hydrate lazy objects
+ D44682487 - Fix tests using `set_lazy_imports(excluding=...)`
+ D44682641 - Split `_imp._maybe_set_parent_attribute function` for adding parent and child side effect separately
+ D45825709 - Save and pass builtins with lazy import objects
+ D49166179 - Fixes `NameError` bug
+ D49522414 - Reduce costs of checking if lazy imports are enabled
Not needed in 3.12 (We use `EAGER_IMPORT_NAME`)
+ D49560412 - Handle error about invalid `__lazy_submodules__`
+ D49647778 - Eager child attributes in excluded modules
+ D52059168  - [CinderX] eliminate special case for strict modules in `_PyObject_LookupAttr` (relevant parts only)

+ Additionally, the environment variable `PYTHONLAZYIMPORTSALL` also turns on Lazy Imports, this also isn't in PEP-690. (`third-party/python/3.12/Python/initconfig.c:1669`)

Reviewed By: itamaro

Differential Revision: D52812410

fbshipit-source-id: a05888e9df8b1adf0dfa479b78ade58a99db2df3

Doc/library/dis.rst

@@ -1152,6 +1152,16 @@ iterations of the loop.
    modifies the namespace.
 
 
+.. opcode:: EAGER_IMPORT_NAME (namei)
+
+   Imports the module ``co_names[namei]`` eagerly (i.e. even if lazy imports are
+   enabled, the module is imported immediately.)  TOS and TOS1 are popped and
+   provide the *fromlist* and *level* arguments of :func:`__import__`.  The
+   module object is pushed onto the stack.  The current namespace is not
+   affected: for a proper import statement, a subsequent :opcode:`STORE_FAST`
+   instruction modifies the namespace.
+
+
 .. opcode:: IMPORT_FROM (namei)
 
    Loads the attribute ``co_names[namei]`` from the module found in ``STACK[-1]``.

Doc/library/sys.rst

@@ -567,6 +567,9 @@ always available.
       * - .. attribute:: flags.safe_path
         - :option:`-P`
 
+      * - .. attribute:: flags.lazy_imports
+        - :option:`-L`
+
       * - .. attribute:: flags.int_max_str_digits
         - :option:`-X int_max_str_digits <-X>`
           (:ref:`integer string conversion length limitation <int_max_str_digits>`)
@@ -600,6 +603,9 @@ always available.
    .. versionchanged:: 3.11
       Added the ``int_max_str_digits`` attribute.
 
+   .. versionchanged:: 3.12
+      Added the ``lazy_imports`` attribute for :option:`-L` option.
+
 
 .. data:: float_info
 

Doc/using/cmdline.rst

@@ -310,6 +310,11 @@ Miscellaneous options
    .. versionadded:: 3.4
 
 
+.. option:: -L
+
+   Enable lazy imports.
+
+
 .. option:: -O
 
    Remove assert statements and any code conditional on the value of

Include/Python.h

@@ -57,6 +57,7 @@
 #include "rangeobject.h"
 #include "memoryobject.h"
 #include "tupleobject.h"
+#include "lazyimportobject.h"
 #include "listobject.h"
 #include "dictobject.h"
 #include "cpython/odictobject.h"

Include/cpython/dictobject.h

@@ -37,6 +37,7 @@ PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key,
 PyAPI_FUNC(PyObject *) _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
 PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
                                                   _Py_Identifier *key);
+PyAPI_FUNC(PyObject *) _PyDict_GetItemKeepLazy(PyObject *dp, PyObject *key);
 PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *);
 PyAPI_FUNC(PyObject *) PyDict_SetDefault(
     PyObject *mp, PyObject *key, PyObject *defaultobj);
@@ -48,6 +49,10 @@ PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key,
                                   int (*predicate)(PyObject *value));
 PyAPI_FUNC(int) _PyDict_Next(
     PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash);
+PyAPI_FUNC(int) PyDict_NextWithError(
+    PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value);
+PyAPI_FUNC(int) _PyDict_NextKeepLazy(
+    PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value);
 
 /* Get the number of items of a dictionary. */
 static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) {
@@ -67,6 +72,9 @@ PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *);
 PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *, PyObject *, PyObject *);
 #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL)
 
+PyAPI_FUNC(Py_ssize_t) PyDict_ResolveLazyImports(PyObject *);
+PyAPI_FUNC(int) PyDict_IsLazyImport(PyObject *mp, PyObject *name);
+
 /* Like PyDict_Merge, but override can be 0, 1 or 2.  If override is 0,
    the first occurrence of a key wins, if override is 1, the last occurrence
    of a key wins, if override is 2, a KeyError with conflicting key as

Include/cpython/import.h

@@ -6,6 +6,7 @@ PyMODINIT_FUNC PyInit__imp(void);
 
 PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *);
 
+PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyThreadState *tstate, PyObject *name);
 PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name);
 PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module);
 PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module);
@@ -26,6 +27,14 @@ PyAPI_FUNC(PyObject *) _Ci_PyImport_CallInitFuncWithContext(const char* context,
                                                             PyObject* (*initfunc)(void));
 // END META PATCH
 
+PyAPI_FUNC(int) _PyImport_IsLazyImportsActive(PyThreadState *tstate);
+
+PyAPI_FUNC(int) PyImport_IsLazyImportsEnabled(void);
+PyAPI_FUNC(PyObject *) PyImport_SetLazyImports(
+    PyObject *enabled, PyObject *excluding);
+PyAPI_FUNC(PyObject *) _PyImport_SetLazyImportsInModule(
+    PyObject *enabled);
+
 struct _inittab {
     const char *name;           /* ASCII encoded string */
     PyObject* (*initfunc)(void);

Include/cpython/initconfig.h

@@ -216,6 +216,8 @@ typedef struct PyConfig {
 
     // If non-zero, we believe we're running from a source tree.
     int _is_python_build;
+
+    int lazy_imports;
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

Include/cpython/pyerrors.h

@@ -176,3 +176,5 @@ extern PyObject *_PyErr_SetImportErrorWithNameFrom(
 
 
 #define Py_FatalError(message) _Py_FatalErrorFunc(__func__, (message))
+
+PyAPI_DATA(PyObject *) PyExc_ImportCycleError;

Include/internal/pycore_ceval.h

@@ -54,6 +54,7 @@ extern void _PyEval_Fini(void);
 
 
 extern PyObject* _PyEval_GetBuiltins(PyThreadState *tstate);
+extern PyObject* _PyEval_GetGlobals(PyThreadState *tstate);
 extern PyObject* _PyEval_BuiltinsFromGlobals(
     PyThreadState *tstate,
     PyObject *globals);

Include/internal/pycore_dict.h

@@ -47,9 +47,13 @@ extern size_t _PyDict_KeysSize(PyDictKeysObject *keys);
  */
 extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr);
 
+/* _Py_dict_lookup_keep_lazy() is the same as _Py_dict_lookup(), but keeps lazy objects unresolved */
+extern Py_ssize_t _Py_dict_lookup_keep_lazy(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr);
+
 extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *);
 extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key);
 extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
+extern PyObject *_PyDict_GetItemKeepLazy(PyObject *, PyObject *);
 
 /* Consumes references to key and value */
 extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value);
@@ -61,6 +65,7 @@ extern PyObject *_PyDict_Pop_KnownHash(PyObject *, PyObject *, Py_hash_t, PyObje
 #define DKIX_DUMMY (-2)  /* Used internally */
 #define DKIX_ERROR (-3)
 #define DKIX_KEY_CHANGED (-4) /* Used internally */
+#define DKIX_VALUE_ERROR (-5) /* Used internally */
 
 typedef enum {
     DICT_KEYS_GENERAL = 0,
@@ -79,7 +84,10 @@ struct _dictkeysobject {
     uint8_t dk_log2_index_bytes;
 
     /* Kind of keys */
-    uint8_t dk_kind;
+    uint8_t dk_kind : 7;
+
+    /* Contains lazy imports */
+    uint8_t dk_lazy_imports : 1;
 
     /* Version number -- Reset to 0 by any modification to keys */
     uint32_t dk_version;

Include/internal/pycore_global_strings.h

@@ -150,6 +150,8 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(__iter__)
         STRUCT_FOR_ID(__itruediv__)
         STRUCT_FOR_ID(__ixor__)
+        STRUCT_FOR_ID(__lazy_imports_enabled__)
+        STRUCT_FOR_ID(__lazy_submodules__)
         STRUCT_FOR_ID(__le__)
         STRUCT_FOR_ID(__len__)
         STRUCT_FOR_ID(__length_hint__)
@@ -404,6 +406,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(exc_value)
         STRUCT_FOR_ID(excepthook)
         STRUCT_FOR_ID(exception)
+        STRUCT_FOR_ID(excluding)
         STRUCT_FOR_ID(existing_file_name)
         STRUCT_FOR_ID(exp)
         STRUCT_FOR_ID(extend)

Include/internal/pycore_import.h

@@ -172,6 +172,37 @@ extern const struct _module_alias * _PyImport_FrozenAliases;
 PyAPI_FUNC(int) _PyImport_CheckSubinterpIncompatibleExtensionAllowed(
     const char *name);
 
+PyAPI_FUNC(PyObject *) _PyImport_ImportName(PyThreadState *tstate,
+                                            PyObject *builtins,
+                                            PyObject *globals,
+                                            PyObject *locals,
+                                            PyObject *name,
+                                            PyObject *fromlist,
+                                            PyObject *level);
+
+PyAPI_FUNC(PyObject *) _PyImport_ImportFrom(PyThreadState *tstate,
+                                            PyObject *v,
+                                            PyObject *name);
+
+PyAPI_FUNC(PyObject *) _PyImport_LazyImportName(PyThreadState *tstate,
+                                                PyObject *builtins,
+                                                PyObject *globals,
+                                                PyObject *locals,
+                                                PyObject *name,
+                                                PyObject *fromlist,
+                                                PyObject *level);
+
+PyAPI_FUNC(PyObject *) _PyImport_LazyImportFrom(PyThreadState *tstate,
+                                                PyObject *v,
+                                                PyObject *name);
+
+PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate(PyThreadState *tstate,
+                                                      PyObject *lazy_import,
+                                                      int full);
+
+PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImport(PyObject *lazy_import,
+                                                int full);
+
 
 // for testing
 PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename);

Include/internal/pycore_interp.h

@@ -230,6 +230,12 @@ struct _is {
 
    /* the initial PyInterpreterState.threads.head */
     PyThreadState _initial_thread;
+
+    // Lazy Imports
+    int lazy_imports;  /* whether lazy imports was enabled at runtime */
+    PyObject *lazy_import_verbose_seen;
+    PyObject *eager_imports;
+    PyObject *lazy_modules;
 };
 
 

Include/internal/pycore_lazyimport.h

@@ -0,0 +1,30 @@
+/* Copyright (c) Meta, Inc. and its affiliates. All Rights Reserved */
+/* File added for Lazy Imports */
+
+#ifndef Py_INTERNAL_LAZYIMPORTOBJECT_H
+#define Py_INTERNAL_LAZYIMPORTOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *lz_builtins;
+    PyObject *lz_from;
+    PyObject *lz_attr;
+} PyLazyImportObject;
+
+
+PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import);
+PyAPI_FUNC(PyObject *) _PyLazyImport_New(PyObject *builtins, PyObject *from, PyObject *attr);
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_LAZYIMPORTOBJECT_H */

Include/lazyimportobject.h

@@ -0,0 +1,20 @@
+/* Copyright (c) Meta, Inc. and its affiliates. All Rights Reserved */
+/* File added for Lazy Imports */
+
+/* Lazy object interface */
+
+#ifndef Py_LAZYIMPORTOBJECT_H
+#define Py_LAZYIMPORTOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_LIMITED_API
+PyAPI_DATA(PyTypeObject) PyLazyImport_Type;
+#define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type)
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_LAZYIMPORTOBJECT_H */