The Import System: importlib, Finders & Loaders

import sys

# sys.modules is the cache
import os
print("os" in sys.modules)        # True — already loaded
print(sys.modules["os"] is os)    # True — same object

# Import resolves to the cached module — zero I/O on repeat imports
import os as os2
print(os is os2)    # True — same module object

# sys.meta_path — the list of finders Python tries in order
for finder in sys.meta_path:
    print(type(finder).__name__)
# BuiltinImporter
# FrozenImporter
# PathFinder

# sys.path — where PathFinder looks for packages/modules
import pprint
pprint.pprint(sys.path)

import sys

# ── Inspect ──
print(len(sys.modules))   # hundreds of modules already loaded at startup

# Find all packages (modules with __path__)
packages = {k for k, v in sys.modules.items()
            if hasattr(v, "__path__")}
print(sorted(packages)[:10])

# ── Force reload ──
import importlib
import mymodule   # noqa (hypothetical)
importlib.reload(mymodule)   # re-executes the module; existing refs still point to old objects

# ── Module substitution (advanced) ──
# Replace a module in sys.modules with a custom object
# Used by mock libraries, lazy import systems

class LazyModule:
    """A placeholder that loads the real module on first attribute access."""
    def __init__(self, name: str):
        object.__setattr__(self, "_name", name)
        object.__setattr__(self, "_module", None)

    def _load(self):
        if object.__getattribute__(self, "_module") is None:
            name = object.__getattribute__(self, "_name")
            import importlib
            module = importlib.import_module(name)
            object.__setattr__(self, "_module", module)
            sys.modules[name] = module   # replace ourselves in the cache
        return object.__getattribute__(self, "_module")

    def __getattr__(self, attr):
        return getattr(self._load(), attr)

# Register a lazy placeholder
sys.modules["json"] = LazyModule("json")
import json                  # returns the LazyModule (no real load yet)
data = json.loads('{"x":1}') # NOW loads — _load() called on first attribute access
print(data)                  # {'x': 1}

import importlib.util, importlib.machinery

# Inspect a real module's spec
spec = importlib.util.find_spec("json")
print(spec.name)          # "json"
print(spec.origin)        # ".../json/__init__.py"
print(spec.submodule_search_locations)  # [".../json"]

# Loader attached to the spec
print(type(spec.loader))  # <class 'importlib.machinery.SourceFileLoader'>
print(spec.loader.path)   # ".../json/__init__.py"

# Manual load via spec
import sys
spec2 = importlib.util.spec_from_file_location("mymod", "/path/to/mymod.py")
module = importlib.util.module_from_spec(spec2)
sys.modules["mymod"] = module
spec2.loader.exec_module(module)
print(module)

import sys
import importlib.abc
import importlib.machinery
import importlib.util
import types

# In-memory "package": name → source code
_VIRTUAL_MODULES = {
    "virtual.greet": """
def hello(name: str) -> str:
    return f"Hello, {name}! (from virtual module)"
""",
    "virtual.math_utils": """
def add(a, b): return a + b
def mul(a, b): return a * b
PI = 3.14159265358979
""",
}


class VirtualFinder(importlib.abc.MetaPathFinder):
    """Finds modules whose source is stored in _VIRTUAL_MODULES."""

    def find_spec(self, fullname, path, target=None):
        if fullname in _VIRTUAL_MODULES:
            source = _VIRTUAL_MODULES[fullname]
            loader = VirtualLoader(fullname, source)
            return importlib.machinery.ModuleSpec(
                name=fullname,
                loader=loader,
                origin=f"<virtual:{fullname}>",
            )
        return None   # not ours — let other finders try


class VirtualLoader(importlib.abc.Loader):
    """Loads a module from an in-memory source string."""

    def __init__(self, fullname: str, source: str):
        self.fullname = fullname
        self.source   = source

    def create_module(self, spec):
        return None   # use default module creation

    def exec_module(self, module):
        code = compile(self.source, f"<virtual:{self.fullname}>", "exec")
        exec(code, module.__dict__)


# Install our finder at the FRONT of sys.meta_path
sys.meta_path.insert(0, VirtualFinder())

# Now import as normal
import virtual.greet as greet
print(greet.hello("World"))      # Hello, World! (from virtual module)

import virtual.math_utils as mu
print(mu.add(3, 4))              # 7
print(mu.PI)                     # 3.14159265358979

# Cleanup
sys.meta_path.pop(0)

import sys
import importlib.abc
import importlib.machinery
import zipfile
import io

class ZipStringFinder(importlib.abc.PathEntryFinder):
    """
    A PathEntryFinder that imports modules from an in-memory zip archive.
    Installed via a sys.path_hooks entry.
    """
    _archives: dict[str, bytes] = {}    # path_entry → zip bytes

    def __init__(self, path_entry: str):
        if path_entry not in self._archives:
            raise ImportError(f"Not a virtual zip: {path_entry}")
        self._zip = zipfile.ZipFile(io.BytesIO(self._archives[path_entry]))

    def find_spec(self, fullname, target=None):
        module_path = fullname.replace(".", "/") + ".py"
        try:
            source = self._zip.read(module_path).decode()
        except KeyError:
            return None
        loader = _ZipStringLoader(fullname, source)
        return importlib.machinery.ModuleSpec(fullname, loader)


class _ZipStringLoader(importlib.abc.Loader):
    def __init__(self, fullname, source):
        self.fullname = fullname
        self.source   = source

    def create_module(self, spec): return None

    def exec_module(self, module):
        exec(compile(self.source, f"<zip:{self.fullname}>", "exec"), module.__dict__)


def zip_path_hook(path_entry: str):
    """Called for each entry in sys.path; raises ImportError if we don't handle it."""
    return ZipStringFinder(path_entry)


# Install the hook
sys.path_hooks.insert(0, zip_path_hook)

# Create a fake in-memory zip archive with a module inside
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
    zf.writestr("zipped_utils.py", "def double(x): return x * 2\n")
ZipStringFinder._archives["virtual_zip://myarchive"] = buf.getvalue()

# Add our virtual path entry
sys.path.insert(0, "virtual_zip://myarchive")
sys.path_importer_cache.clear()   # clear cached PathFinder decisions

import zipped_utils
print(zipped_utils.double(21))   # 42

import sys, types, importlib.util

# Inspect a namespace package
spec = importlib.util.find_spec("mypkg")   # assuming it exists
if spec and spec.submodule_search_locations:
    print(f"Namespace package paths: {list(spec.submodule_search_locations)}")
    # ['/path/A/mypkg', '/path/B/mypkg']

# Create a synthetic namespace package programmatically
ns_pkg = types.ModuleType("synthetic_ns")
ns_pkg.__path__ = ["/some/path/a", "/some/path/b"]   # no __file__
ns_pkg.__package__ = "synthetic_ns"
ns_pkg.__spec__ = None
sys.modules["synthetic_ns"] = ns_pkg
print(ns_pkg)   # <module 'synthetic_ns' (namespace)>

import sys
import importlib.abc
import importlib.machinery
import importlib.util
import ast
import textwrap


class LoggingTransformer(ast.NodeTransformer):
    """
    AST transformer: inject a print statement at the start of every function.
    Demonstrates import-time code transformation.
    """
    def visit_FunctionDef(self, node):
        self.generic_visit(node)   # recurse into nested functions
        log_stmt = ast.parse(
            f'print(f"CALL: {node.name}()")',
            mode="single",
        ).body[0]
        ast.copy_location(log_stmt, node)
        node.body.insert(0, log_stmt)
        return node

    visit_AsyncFunctionDef = visit_FunctionDef


class TransformingLoader(importlib.abc.SourceLoader):
    """Applies LoggingTransformer to any module we load."""

    def __init__(self, fullname: str, path: str):
        self.fullname = fullname
        self.path     = path

    def get_data(self, path: bytes | str) -> bytes:
        with open(path, "rb") as f:
            return f.read()

    def get_filename(self, fullname: str) -> str:
        return self.path

    def source_to_code(self, data, path, *, _optimize=-1):
        source = data.decode("utf-8") if isinstance(data, bytes) else data
        tree   = ast.parse(source, path)
        tree   = LoggingTransformer().visit(tree)
        ast.fix_missing_locations(tree)
        return compile(tree, path, "exec", optimize=_optimize)


class TransformingFinder(importlib.abc.MetaPathFinder):
    """Intercepts imports of modules whose names end with '_debug'."""

    def find_spec(self, fullname, path, target=None):
        if not fullname.endswith("_debug"):
            return None
        # Strip the _debug suffix to find the real file
        real_name = fullname[:-6]   # e.g. "utils_debug" → "utils"
        real_spec = importlib.util.find_spec(real_name)
        if real_spec is None or real_spec.origin is None:
            return None
        loader = TransformingLoader(fullname, real_spec.origin)
        return importlib.machinery.ModuleSpec(fullname, loader, origin=real_spec.origin)


# Install the finder
sys.meta_path.insert(0, TransformingFinder())

# Now: import utils_debug loads utils.py with logging injected
# import utils_debug   # every function will print its name when called

# Package structure:
# myapp/
#   __init__.py
#   utils.py
#   api/
#     __init__.py
#     routes.py
#     auth.py

# Inside myapp/api/routes.py:
from .auth import verify_token        # sibling: myapp.api.auth
from ..utils import format_response   # parent: myapp.utils
from . import auth                    # import sibling module object

# NEVER use relative imports in scripts run directly (not as packages)
# python myapp/api/routes.py   → ImportError: attempted relative import with no known parent package
# python -m myapp.api.routes   → works, because -m sets __package__ correctly

# __package__ and __name__ control relative import resolution
import myapp.api.routes as r
print(r.__name__)     # "myapp.api.routes"
print(r.__package__)  # "myapp.api"  ← used to resolve relative dots

# myapp/__init__.py — controls what `import myapp` exposes

# 1. Eager re-export: make sub-module symbols available at package level
from .utils import format_response, parse_config   # now: from myapp import format_response

# 2. Lazy sub-package loading: only import when needed
def get_api():
    from .api import routes   # imported on first call, not at package load time
    return routes

# 3. __all__: controls `from myapp import *`
__all__ = ["format_response", "parse_config"]

# 4. Version
__version__ = "2.1.0"

# 5. Prevent direct sub-module access before __init__ runs
# (sub-modules are NOT automatically accessible as attributes of the package
#  unless they are imported in __init__.py or accessed with dot notation first)

import myapp
# myapp.utils is None / AttributeError  ← unless explicitly imported in __init__.py
import myapp.utils
# NOW myapp.utils is accessible — importing a sub-module always sets it as an attribute

# ── Why circular imports fail ──
# a.py: from b import B
# b.py: from a import A   ← A not yet defined when b is executing

# The sequence:
# 1. import a
# 2. a.py starts executing: "from b import B"
# 3. b.py starts executing: "from a import A"
# 4. sys.modules["a"] exists (partial!) but A is not yet defined in it
# 5. ImportError: cannot import name 'A' from partially initialized module 'a'

# ── Fix 1: Import at function level (deferred import) ──
# a.py
class A:
    def get_b(self):
        from b import B    # imported when the method is called, not at module load
        return B()

# ── Fix 2: Import the module, not the name ──
# a.py
import b   # just the module — works because sys.modules["b"] is added immediately
class A:
    def get_b(self): return b.B()   # b.B is resolved at call time

# ── Fix 3: Restructure — extract shared code to a third module ──
# common.py: shared types / interfaces
# a.py: import from common
# b.py: import from common
# → no cycle

# ── Detecting circular imports ──
import sys

def find_import_chain(target: str) -> list:
    """Return the import chain leading to target (very simplified)."""
    return [name for name in sys.modules if target in name]

# importlib.util.find_spec raises ImportError on circular deps
# Use modulefinder for static analysis:
from modulefinder import ModuleFinder
finder = ModuleFinder()
finder.run_script("main.py")   # analyses all imports
for name, mod in finder.modules.items():
    if mod.__file__:
        print(f"  {name}: {mod.__file__}")

import importlib.util
import sys

# ── 1. Import a module from an arbitrary file path ──
def import_from_path(module_name: str, file_path: str):
    spec   = importlib.util.spec_from_file_location(module_name, file_path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module    # register before exec so circular imports work
    spec.loader.exec_module(module)
    return module

plugin = import_from_path("my_plugin", "/path/to/plugin.py")

# ── 2. Check if a module is importable without importing it ──
def is_importable(name: str) -> bool:
    return importlib.util.find_spec(name) is not None

print(is_importable("json"))      # True
print(is_importable("nonexist"))  # False

# ── 3. Get the file path of an installed module ──
spec = importlib.util.find_spec("numpy")
if spec:
    print(spec.origin)   # /path/to/numpy/__init__.py

# ── 4. Conditional import with a fallback ──
try:
    import ujson as json_lib
except ImportError:
    import json as json_lib

# Or using importlib:
def import_first_available(*names: str):
    for name in names:
        if importlib.util.find_spec(name):
            return importlib.import_module(name)
    raise ImportError(f"None of {names} could be imported")

json_lib = import_first_available("ujson", "orjson", "json")

# ── 5. Reload with dependency tracking ──
import importlib, sys

def deep_reload(module_name: str, prefix: str) -> None:
    """Reload a module and all sub-modules sharing the given prefix."""
    to_reload = [
        name for name in list(sys.modules)
        if name == module_name or name.startswith(prefix + ".")
    ]
    for name in sorted(to_reload, reverse=True):  # children before parents
        mod = sys.modules.pop(name, None)
        if mod:
            importlib.import_module(name)   # re-import into fresh sys.modules entry