meerschaum.utils.packages

   1#! /usr/bin/env python
   2# -*- coding: utf-8 -*-
   3# vim:fenc=utf-8
   4
   5"""
   6Functions for managing packages and virtual environments reside here.
   7"""
   8
   9from __future__ import annotations
  10
  11import importlib.util, os, pathlib, re
  12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
  13from meerschaum.utils.threading import Lock, RLock
  14from meerschaum.utils.packages._packages import (
  15    packages,
  16    all_packages,
  17    get_install_names,
  18    _MRSM_PACKAGE_ARCHIVES_PREFIX,
  19)
  20from meerschaum.utils.venv import (
  21    activate_venv,
  22    deactivate_venv,
  23    venv_executable,
  24    venv_exec,
  25    venv_exists,
  26    venv_target_path,
  27    inside_venv,
  28    Venv,
  29    init_venv,
  30)
  31
  32_import_module = importlib.import_module
  33_import_hook_venv = None
  34_locks = {
  35    '_pkg_resources_get_distribution': RLock(),
  36    'import_versions': RLock(),
  37    '_checked_for_updates': RLock(),
  38    '_is_installed_first_check': RLock(),
  39    'emitted_pandas_warning': RLock(),
  40    'emitted_auto_install_warning': RLock(),
  41    '_install_thread_locks': RLock(),
  42}
  43_checked_for_updates = set()
  44_is_installed_first_check: Dict[Tuple[str, Optional[str], bool, bool], bool] = {}
  45_install_thread_locks = {}
  46_install_lock_depths = {}
  47emitted_auto_install_warning = False
  48
  49
  50def _get_pip_install_target_path(venv: Optional[str] = 'mrsm') -> pathlib.Path:
  51    """Resolve an installation target without creating it."""
  52    if venv is not None:
  53        return venv_target_path(venv, allow_nonexistent=True)
  54
  55    import sys
  56    return pathlib.Path(sys.prefix)
  57
  58
  59def get_pip_install_lock_path(venv: Optional[str] = 'mrsm') -> pathlib.Path:
  60    """Return the cross-process package installation lock path for an environment."""
  61    import hashlib
  62    import tempfile
  63    target_path = _get_pip_install_target_path(venv).resolve()
  64    target_key = os.path.normcase(str(target_path))
  65    target_hash = hashlib.sha256(target_key.encode('utf-8')).hexdigest()[:16]
  66    return (
  67        pathlib.Path(tempfile.gettempdir())
  68        / 'meerschaum-package-installs'
  69        / (target_hash + '.lock')
  70    )
  71
  72
  73def _pip_install_lock(venv: Optional[str] = 'mrsm'):
  74    """Serialize package mutations across threads and processes for an environment."""
  75    from contextlib import contextmanager
  76
  77    @contextmanager
  78    def _locked():
  79        from threading import get_ident
  80        from meerschaum.utils.locks import InterProcessLock
  81
  82        lock_path = get_pip_install_lock_path(venv)
  83        lock_key = str(lock_path)
  84        with _locks['_install_thread_locks']:
  85            thread_lock = _install_thread_locks.setdefault(lock_key, RLock())
  86
  87        with thread_lock:
  88            depth_key = (get_ident(), lock_key)
  89            depth = _install_lock_depths.get(depth_key, 0)
  90            _install_lock_depths[depth_key] = depth + 1
  91            try:
  92                if depth:
  93                    yield
  94                    return
  95                lock_path.parent.mkdir(parents=True, exist_ok=True)
  96                with InterProcessLock(lock_path):
  97                    yield
  98            finally:
  99                if depth:
 100                    _install_lock_depths[depth_key] -= 1
 101                else:
 102                    _install_lock_depths.pop(depth_key, None)
 103
 104    return _locked()
 105
 106
 107def _locked_pip_install(func):
 108    """Lock package mutations while preserving `pip_install()`'s public signature."""
 109    import functools
 110
 111    @functools.wraps(func)
 112    def _wrapped(*args, **kw):
 113        if kw.get('dry_run', False):
 114            return func(*args, **kw)
 115        with _pip_install_lock(kw.get('venv', 'mrsm')):
 116            return func(*args, **kw)
 117
 118    return _wrapped
 119
 120
 121def get_module_path(
 122    import_name: str,
 123    venv: Optional[str] = 'mrsm',
 124    debug: bool = False,
 125    _try_install_name_on_fail: bool = True,
 126) -> Union[pathlib.Path, None]:
 127    """
 128    Get a module's path without importing.
 129    """
 130    import site
 131    if debug:
 132        from meerschaum.utils.debug import dprint
 133    if not _try_install_name_on_fail:
 134        install_name = _import_to_install_name(import_name, with_version=False)
 135        install_name_lower = install_name.lower().replace('-', '_')
 136        import_name_lower = install_name_lower
 137    else:
 138        import_name_lower = import_name.lower().replace('-', '_')
 139
 140    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 141    if not vtp.exists():
 142        if debug:
 143            dprint(
 144                (
 145                    "Venv '{venv}' does not exist, cannot import "
 146                    + f"'{import_name}'."
 147                ),
 148                color = False,
 149            )
 150        return None
 151
 152    venv_target_candidate_paths = [vtp]
 153    if venv is None:
 154        site_user_packages_dirs = [
 155            pathlib.Path(site.getusersitepackages())
 156        ] if not inside_venv() else []
 157        site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
 158
 159        paths_to_add = [
 160            path
 161            for path in site_user_packages_dirs + site_packages_dirs
 162            if path not in venv_target_candidate_paths
 163        ]
 164        venv_target_candidate_paths += paths_to_add
 165
 166    candidates = []
 167    for venv_target_candidate in venv_target_candidate_paths:
 168        try:
 169            file_names = os.listdir(venv_target_candidate)
 170        except FileNotFoundError:
 171            continue
 172        for file_name in file_names:
 173            file_name_lower = file_name.lower().replace('-', '_')
 174            if not file_name_lower.startswith(import_name_lower):
 175                continue
 176            if file_name.endswith('dist_info'):
 177                continue
 178            file_path = venv_target_candidate / file_name
 179
 180            ### Most likely: Is a directory with __init__.py
 181            if file_name_lower == import_name_lower and file_path.is_dir():
 182                init_path = file_path / '__init__.py'
 183                if init_path.exists():
 184                    candidates.append(init_path)
 185
 186            ### May be a standalone .py file.
 187            elif file_name_lower == import_name_lower + '.py':
 188                candidates.append(file_path)
 189
 190            ### Compiled wheels (e.g. pyodbc)
 191            elif file_name_lower.startswith(import_name_lower + '.'):
 192                candidates.append(file_path)
 193
 194    if len(candidates) == 1:
 195        return candidates[0]
 196
 197    if not candidates:
 198        if _try_install_name_on_fail:
 199            return get_module_path(
 200                import_name, venv=venv, debug=debug,
 201                _try_install_name_on_fail=False
 202            )
 203        return None
 204
 205    specs_paths = []
 206    for candidate_path in candidates:
 207        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
 208        if spec is not None:
 209            return candidate_path
 210    
 211    return None
 212
 213
 214def manually_import_module(
 215    import_name: str,
 216    venv: Optional[str] = 'mrsm',
 217    check_update: bool = True,
 218    check_pypi: bool = False,
 219    install: bool = True,
 220    split: bool = True,
 221    warn: bool = True,
 222    color: bool = True,
 223    debug: bool = False,
 224    use_sys_modules: bool = True,
 225) -> Union['ModuleType', None]:
 226    """
 227    Manually import a module from a virtual environment (or the base environment).
 228
 229    Parameters
 230    ----------
 231    import_name: str
 232        The name of the module.
 233        
 234    venv: Optional[str], default 'mrsm'
 235        The virtual environment to read from.
 236
 237    check_update: bool, default True
 238        If `True`, examine whether the available version of the package meets the required version.
 239
 240    check_pypi: bool, default False
 241        If `True`, check PyPI for updates before importing.
 242
 243    install: bool, default True
 244        If `True`, install the package if it's not installed or needs an update.
 245
 246    split: bool, default True
 247        If `True`, split `import_name` on periods to get the package name.
 248
 249    warn: bool, default True
 250        If `True`, raise a warning if the package cannot be imported.
 251
 252    color: bool, default True
 253        If `True`, use color output for debug and warning text.
 254
 255    debug: bool, default False
 256        Verbosity toggle.
 257
 258    use_sys_modules: bool, default True
 259        If `True`, return the module in `sys.modules` if it exists.
 260        Otherwise continue with manually importing.
 261
 262    Returns
 263    -------
 264    The specified module or `None` if it can't be imported.
 265
 266    """
 267    import sys
 268    _previously_imported = import_name in sys.modules
 269    if _previously_imported and use_sys_modules:
 270        return sys.modules[import_name]
 271
 272    from meerschaum.utils.warnings import warn as warn_function
 273    import warnings
 274    root_name = import_name.split('.')[0] if split else import_name
 275    install_name = _import_to_install_name(root_name)
 276
 277    root_path = get_module_path(root_name, venv=venv)
 278    if root_path is None:
 279        return None
 280
 281    mod_path = root_path
 282    if mod_path.is_dir():
 283        for _dir in import_name.split('.')[:-1]:
 284            mod_path = mod_path / _dir
 285            possible_end_module_filename = import_name.split('.')[-1] + '.py'
 286            try:
 287                mod_path = (
 288                    (mod_path / possible_end_module_filename)
 289                    if possible_end_module_filename in os.listdir(mod_path)
 290                    else (
 291                        mod_path / import_name.split('.')[-1] / '__init__.py'
 292                    )
 293                )
 294            except Exception:
 295                mod_path = None
 296
 297    spec = (
 298        importlib.util.find_spec(import_name)
 299        if mod_path is None or not mod_path.exists()
 300        else importlib.util.spec_from_file_location(import_name, str(mod_path))
 301    )
 302    root_spec = (
 303        importlib.util.find_spec(root_name)
 304        if not root_path.exists()
 305        else importlib.util.spec_from_file_location(root_name, str(root_path))
 306    )
 307
 308    ### Check for updates before importing.
 309    _version = (
 310        determine_version(
 311            pathlib.Path(root_spec.origin),
 312            import_name=root_name, venv=venv, debug=debug
 313        ) if root_spec is not None and root_spec.origin is not None else None
 314    )
 315
 316    if _version is not None:
 317        if check_update:
 318            if need_update(
 319                None,
 320                import_name=root_name,
 321                version=_version,
 322                check_pypi=check_pypi,
 323                debug=debug,
 324            ):
 325                if install:
 326                    if not pip_install(
 327                        root_name,
 328                        venv=venv,
 329                        split=False,
 330                        check_update=check_update,
 331                        color=color,
 332                        debug=debug
 333                    ) and warn:
 334                        warn_function(
 335                            f"There's an update available for '{install_name}', "
 336                            + "but it failed to install. "
 337                            + "Try installig via Meerschaum with "
 338                            + "`install packages '{install_name}'`.",
 339                            ImportWarning,
 340                            stacklevel=3,
 341                            color=False,
 342                        )
 343                elif warn:
 344                    warn_function(
 345                        f"There's an update available for '{root_name}'.",
 346                        stack=False,
 347                        color=False,
 348                    )
 349                spec = (
 350                    importlib.util.find_spec(import_name)
 351                    if mod_path is None or not mod_path.exists()
 352                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
 353                )
 354
 355    if spec is None:
 356        try:
 357            mod = _import_module(import_name)
 358        except Exception:
 359            mod = None
 360        return mod
 361
 362    with Venv(venv, debug=debug):
 363        mod = importlib.util.module_from_spec(spec)
 364        old_sys_mod = sys.modules.get(import_name, None)
 365        sys.modules[import_name] = mod
 366
 367        try:
 368            with warnings.catch_warnings():
 369                warnings.filterwarnings('ignore', 'The NumPy')
 370                spec.loader.exec_module(mod)
 371        except Exception:
 372            pass
 373        mod = _import_module(import_name)
 374        if old_sys_mod is not None:
 375            sys.modules[import_name] = old_sys_mod
 376        else:
 377            del sys.modules[import_name]
 378
 379    return mod
 380
 381
 382def _import_to_install_name(import_name: str, with_version: bool = True) -> str:
 383    """
 384    Try to translate an import name to an installation name.
 385    """
 386    install_name = all_packages.get(import_name, import_name)
 387    if with_version:
 388        return install_name
 389    return get_install_no_version(install_name)
 390
 391
 392def _import_to_dir_name(import_name: str) -> str:
 393    """
 394    Translate an import name to the package name in the sites-packages directory.
 395    """
 396    import re
 397    return re.split(
 398        r'[<>=\[]', all_packages.get(import_name, import_name)
 399    )[0].replace('-', '_').lower() 
 400
 401
 402def _install_to_import_name(install_name: str) -> str:
 403    """
 404    Translate an installation name to a package's import name.
 405    """
 406    _install_no_version = get_install_no_version(install_name)
 407    return get_install_names().get(_install_no_version, _install_no_version)
 408
 409
 410def get_install_no_version(install_name: str) -> str:
 411    """
 412    Strip the version information from the install name.
 413    """
 414    import re
 415    return re.split(r'[\[=<>,! \]]', install_name)[0]
 416
 417
 418import_versions = {}
 419def determine_version(
 420    path: pathlib.Path,
 421    import_name: Optional[str] = None,
 422    venv: Optional[str] = 'mrsm',
 423    search_for_metadata: bool = True,
 424    split: bool = True,
 425    warn: bool = False,
 426    debug: bool = False,
 427) -> Union[str, None]:
 428    """
 429    Determine a module's `__version__` string from its filepath.
 430    
 431    First it searches for pip metadata, then it attempts to import the module in a subprocess.
 432
 433    Parameters
 434    ----------
 435    path: pathlib.Path
 436        The file path of the module.
 437
 438    import_name: Optional[str], default None
 439        The name of the module. If omitted, it will be determined from the file path.
 440        Defaults to `None`.
 441
 442    venv: Optional[str], default 'mrsm'
 443        The virtual environment of the Python interpreter to use if importing is necessary.
 444
 445    search_for_metadata: bool, default True
 446        If `True`, search the pip site_packages directory (assumed to be the parent)
 447        for the corresponding dist-info directory.
 448
 449    warn: bool, default True
 450        If `True`, raise a warning if the module fails to import in the subprocess.
 451
 452    split: bool, default True
 453        If `True`, split the determined import name by periods to get the room name.
 454
 455    Returns
 456    -------
 457    The package's version string if available or `None`.
 458    If multiple versions are found, it will trigger an import in a subprocess.
 459
 460    """
 461    with _locks['import_versions']:
 462        if venv not in import_versions:
 463            import_versions[venv] = {}
 464    import os
 465    old_cwd = os.getcwd()
 466    from meerschaum.utils.warnings import warn as warn_function
 467    if import_name is None:
 468        import_name = path.parent.stem if path.stem == '__init__' else path.stem
 469        import_name = import_name.split('.')[0] if split else import_name
 470    if import_name in import_versions[venv]:
 471        return import_versions[venv][import_name]
 472    _version = None
 473    module_parent_dir = (
 474        path.parent.parent if path.stem == '__init__' else path.parent
 475    ) if path is not None else venv_target_path(venv, allow_nonexistent=True, debug=debug)
 476
 477    if not module_parent_dir.exists():
 478        return None
 479
 480    installed_dir_name = _import_to_dir_name(import_name)
 481    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
 482
 483    ### First, check if a dist-info directory exists.
 484    _found_versions = []
 485    if search_for_metadata:
 486        try:
 487            filenames = os.listdir(module_parent_dir)
 488        except FileNotFoundError:
 489            filenames = []
 490        for filename in filenames:
 491            if not filename.endswith('.dist-info'):
 492                continue
 493            filename_lower = filename.lower()
 494            if not filename_lower.startswith(clean_installed_dir_name + '-'):
 495                continue
 496            _v = filename.replace('.dist-info', '').split("-")[-1]
 497            _found_versions.append(_v)
 498
 499    if len(_found_versions) == 1:
 500        _version = _found_versions[0]
 501        with _locks['import_versions']:
 502            import_versions[venv][import_name] = _version
 503        return _found_versions[0]
 504
 505    if not _found_versions:
 506        try:
 507            import importlib.metadata as importlib_metadata
 508        except ImportError:
 509            importlib_metadata = attempt_import(
 510                'importlib_metadata',
 511                debug=debug, check_update=False, precheck=False,
 512                color=False, check_is_installed=False, lazy=False,
 513            )
 514        try:
 515            os.chdir(module_parent_dir)
 516            _version = importlib_metadata.metadata(import_name)['Version']
 517        except Exception:
 518            _version = None
 519        finally:
 520            os.chdir(old_cwd)
 521
 522        if _version is not None:
 523            with _locks['import_versions']:
 524                import_versions[venv][import_name] = _version
 525            return _version
 526
 527    if debug:
 528        print(f'Found multiple versions for {import_name}: {_found_versions}')
 529
 530    module_parent_dir_str = module_parent_dir.as_posix()
 531
 532    ### Not a pip package, so let's try importing the module directly (in a subprocess).
 533    _no_version_str = 'no-version'
 534    code = (
 535        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
 536        + f"module = importlib.import_module('{import_name}');\n"
 537        + "try:\n"
 538        + "  print(module.__version__ , end='')\n"
 539        + "except:\n"
 540        + f"  print('{_no_version_str}', end='')"
 541    )
 542    exit_code, stdout_bytes, stderr_bytes = venv_exec(
 543        code, venv=venv, with_extras=True, debug=debug
 544    )
 545    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
 546    _version = stdout.split('\n')[-1] if exit_code == 0 else None
 547    _version = _version if _version != _no_version_str else None
 548
 549    if _version is None:
 550        _version = _get_package_metadata(import_name, venv).get('version', None)
 551    if _version is None and warn:
 552        warn_function(
 553            f"Failed to determine a version for '{import_name}':\n{stderr}",
 554            stack = False
 555        )
 556
 557    ### If `__version__` doesn't exist, return `None`.
 558    import_versions[venv][import_name] = _version
 559    return _version
 560
 561
 562def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]:
 563    """
 564    Get a package's metadata from pip.
 565    This is useful for getting a version when no `__version__` is defined
 566    and multiple versions are installed.
 567
 568    Parameters
 569    ----------
 570    import_name: str
 571        The package's import or installation name.
 572
 573    venv: Optional[str]
 574        The virtual environment which contains the package.
 575
 576    Returns
 577    -------
 578    A dictionary of metadata from pip.
 579    """
 580    import meerschaum.config.paths as paths
 581    install_name = _import_to_install_name(import_name)
 582    if install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX):
 583        return {}
 584    _args = ['pip', 'show', install_name]
 585    if venv is not None:
 586        cache_dir_path = paths.VIRTENV_RESOURCES_PATH / venv / 'cache'
 587        _args += ['--cache-dir', cache_dir_path.as_posix()]
 588
 589    if use_uv():
 590        package_name = 'uv'
 591        _args = ['pip', 'show', install_name]
 592    else:
 593        package_name = 'pip'
 594        _args = ['show', install_name]
 595
 596    proc = run_python_package(
 597        package_name, _args,
 598        capture_output=True, as_proc=True, venv=venv, universal_newlines=True,
 599    )
 600    outs, errs = proc.communicate()
 601    lines = outs.split('\n')
 602    meta = {}
 603    for line in lines:
 604        vals = line.split(": ")
 605        if len(vals) != 2:
 606            continue
 607        k, v = vals[0].lower(), vals[1]
 608        if v and 'UNKNOWN' not in v:
 609            meta[k] = v
 610    return meta
 611
 612
 613def need_update(
 614    package: Optional['ModuleType'] = None,
 615    install_name: Optional[str] = None,
 616    import_name: Optional[str] = None,
 617    version: Optional[str] = None,
 618    check_pypi: bool = False,
 619    split: bool = True,
 620    color: bool = True,
 621    debug: bool = False,
 622    _run_determine_version: bool = True,
 623) -> bool:
 624    """
 625    Check if a Meerschaum dependency needs an update.
 626    Returns a bool for whether or not a package needs to be updated.
 627
 628    Parameters
 629    ----------
 630    package: 'ModuleType'
 631        The module of the package to be updated.
 632
 633    install_name: Optional[str], default None
 634        If provided, use this string to determine the required version.
 635        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
 636
 637    import_name:
 638        If provided, override the package's `__name__` string.
 639
 640    version: Optional[str], default None
 641        If specified, override the package's `__version__` string.
 642
 643    check_pypi: bool, default False
 644        If `True`, check pypi.org for updates.
 645        Defaults to `False`.
 646
 647    split: bool, default True
 648        If `True`, split the module's name on periods to detrive the root name.
 649        Defaults to `True`.
 650
 651    color: bool, default True
 652        If `True`, format debug output.
 653        Defaults to `True`.
 654
 655    debug: bool, default True
 656        Verbosity toggle.
 657
 658    Returns
 659    -------
 660    A bool indicating whether the package requires an update.
 661
 662    """
 663    if debug:
 664        from meerschaum.utils.debug import dprint
 665    from meerschaum.utils.warnings import warn as warn_function
 666    import re
 667    root_name = (
 668        package.__name__.split('.')[0] if split else package.__name__
 669    ) if import_name is None else (
 670        import_name.split('.')[0] if split else import_name
 671    )
 672    install_name = install_name or _import_to_install_name(root_name)
 673    with _locks['_checked_for_updates']:
 674        if install_name in _checked_for_updates:
 675            return False
 676        _checked_for_updates.add(install_name)
 677
 678    _install_no_version = get_install_no_version(install_name)
 679    required_version = (
 680        install_name
 681        .replace(_install_no_version, '')
 682    )
 683    if ']' in required_version:
 684        required_version = required_version.split(']')[1]
 685
 686    ### No minimum version was specified, and we're not going to check PyPI.
 687    if not required_version and not check_pypi:
 688        return False
 689
 690    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
 691    if '.dev' in required_version:
 692        required_version = required_version.split('.dev')[0]
 693    if version and '.dev' in version:
 694        version = version.split('.dev')[0]
 695
 696    try:
 697        if not version:
 698            if not _run_determine_version:
 699                version = determine_version(
 700                    pathlib.Path(package.__file__),
 701                    import_name=root_name, warn=False, debug=debug
 702                )
 703        if version is None:
 704            return False
 705    except Exception as e:
 706        if debug:
 707            dprint(str(e), color=color)
 708            dprint("No version could be determined from the installed package.", color=color)
 709        return False
 710    split_version = version.split('.')
 711    last_part = split_version[-1]
 712    if len(split_version) == 2:
 713        version = '.'.join(split_version) + '.0'
 714    elif 'dev' in last_part or 'rc' in last_part:
 715        tag = 'dev' if 'dev' in last_part else 'rc'
 716        last_sep = '-'
 717        if not last_part.startswith(tag):
 718            last_part = f'-{tag}'.join(last_part.split(tag))
 719            last_sep = '.'
 720        version = '.'.join(split_version[:-1]) + last_sep + last_part
 721    elif len(split_version) > 3:
 722        version = '.'.join(split_version[:3])
 723
 724    packaging_version = attempt_import(
 725        'packaging.version', check_update=False, lazy=False, debug=debug,
 726    )
 727
 728    ### Get semver if necessary
 729    if required_version:
 730        semver_path = get_module_path('semver', debug=debug)
 731        if semver_path is None:
 732            no_venv_semver_path = get_module_path('semver', venv=None, debug=debug)
 733            if no_venv_semver_path is None:
 734                pip_install(_import_to_install_name('semver'), debug=debug)
 735        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
 736    if check_pypi:
 737        ### Check PyPI for updates
 738        update_checker = attempt_import(
 739            'update_checker', lazy=False, check_update=False, debug=debug
 740        )
 741        checker = update_checker.UpdateChecker()
 742        result = checker.check(_install_no_version, version)
 743    else:
 744        ### Skip PyPI and assume we can't be sure.
 745        result = None
 746
 747    ### Compare PyPI's version with our own.
 748    if result is not None:
 749        ### We have a result from PyPI and a stated required version.
 750        if required_version:
 751            try:
 752                return semver.Version.parse(result.available_version).match(required_version)
 753            except AttributeError as e:
 754                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 755                semver = manually_import_module('semver', venv='mrsm')
 756                return semver.Version.parse(version).match(required_version)
 757            except Exception as e:
 758                if debug:
 759                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
 760                return False
 761
 762        ### If `check_pypi` and we don't have a required version, check if PyPI's version
 763        ### is newer than the installed version.
 764        else:
 765            return (
 766                packaging_version.parse(result.available_version) > 
 767                packaging_version.parse(version)
 768            )
 769
 770    ### We might be depending on a prerelease.
 771    ### Sanity check that the required version is not greater than the installed version. 
 772    required_version = (
 773        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
 774        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
 775        .replace('-py3-none-any.whl', '')
 776    )
 777
 778    if 'a' in required_version:
 779        required_version = required_version.replace('a', '-pre.').replace('+mrsm', '')
 780        version = version.replace('a', '-pre.').replace('+mrsm', '')
 781    try:
 782        return (
 783            (not semver.Version.parse(version).match(required_version))
 784            if required_version else False
 785        )
 786    except AttributeError:
 787        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 788        semver = manually_import_module('semver', venv='mrsm', debug=debug)
 789        return (
 790            (not semver.Version.parse(version).match(required_version))
 791            if required_version else False
 792        )
 793    except Exception as e:
 794        print(f"Unable to parse version ({version}) for package '{import_name}'.")
 795        print(e)
 796        if debug:
 797            dprint(e)
 798        return False
 799    try:
 800        return (
 801            packaging_version.parse(version) > 
 802            packaging_version.parse(required_version)
 803        )
 804    except Exception as e:
 805        if debug:
 806            dprint(e)
 807        return False
 808    return False
 809
 810
 811def get_pip(
 812    venv: Optional[str] = 'mrsm',
 813    color: bool = True,
 814    debug: bool = False,
 815) -> bool:
 816    """
 817    Download and run the get-pip.py script.
 818
 819    Parameters
 820    ----------
 821    venv: Optional[str], default 'mrsm'
 822        The virtual environment into which to install `pip`.
 823
 824    color: bool, default True
 825        If `True`, force color output.
 826
 827    debug: bool, default False
 828        Verbosity toggle.
 829
 830    Returns
 831    -------
 832    A bool indicating success.
 833
 834    """
 835    import sys
 836    import subprocess
 837    import meerschaum.config.paths as paths
 838    from meerschaum.utils.misc import wget
 839    from meerschaum._internal.static import STATIC_CONFIG
 840    url = STATIC_CONFIG['system']['urls']['get-pip.py']
 841    dest = paths.CACHE_RESOURCES_PATH / 'get-pip.py'
 842    try:
 843        wget(url, dest, color=False, debug=debug)
 844    except Exception:
 845        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
 846        sys.exit(1)
 847    if venv is not None:
 848        init_venv(venv=venv, debug=debug)
 849    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
 850    return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0
 851
 852
 853def get_pip_install_plan(
 854    *install_names: str,
 855    args: Optional[List[str]] = None,
 856    requirements_file_path: Union[pathlib.Path, str, None] = None,
 857    venv: Optional[str] = 'mrsm',
 858    _uninstall: bool = False,
 859    _install_uv_pip: bool = True,
 860    _use_uv_pip: bool = True,
 861    debug: bool = False,
 862) -> Dict[str, Any]:
 863    """Return a read-only summary of a package installation request."""
 864    try:
 865        import pip  # noqa: F401
 866        have_pip = True
 867    except ImportError:
 868        have_pip = venv_contains_package('pip', venv=None, debug=debug)
 869
 870    try:
 871        import uv
 872        uv_bin = str(uv.find_uv_bin())
 873    except (ImportError, FileNotFoundError):
 874        uv_bin = None
 875
 876    use_uv_pip = bool(
 877        _use_uv_pip
 878        and uv_bin
 879        and venv is not None
 880        and is_uv_enabled()
 881    )
 882    requested_packages = [
 883        (
 884            get_install_no_version(install_name)
 885            if _uninstall or install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX)
 886            else install_name
 887        )
 888        for install_name in install_names
 889    ]
 890    auto_install_enabled = os.environ.get('MRSM_NO_AUTO_INSTALL', '').lower() not in (
 891        '1', 'true', 'yes'
 892    )
 893    return {
 894        'operation': 'uninstall' if _uninstall else 'install',
 895        'environment': venv,
 896        'target': str(_get_pip_install_target_path(venv)),
 897        'packages': requested_packages,
 898        'requirements_file': (
 899            str(pathlib.Path(requirements_file_path).resolve())
 900            if requirements_file_path is not None
 901            else None
 902        ),
 903        'args': list(args if args is not None else ([] if _uninstall else ['--upgrade'])),
 904        'installer': 'uv' if use_uv_pip else 'pip',
 905        'installer_available': bool(use_uv_pip or have_pip),
 906        'pip_fallback': bool(use_uv_pip and have_pip),
 907        'auto_install_enabled': auto_install_enabled,
 908        'would_bootstrap_uv': bool(
 909            not use_uv_pip
 910            and not uv_bin
 911            and have_pip
 912            and _install_uv_pip
 913            and is_uv_enabled()
 914        ),
 915        'lock_path': str(get_pip_install_lock_path(venv)),
 916    }
 917
 918
 919@_locked_pip_install
 920def pip_install(
 921    *install_names: str,
 922    args: Optional[List[str]] = None,
 923    requirements_file_path: Union[pathlib.Path, str, None] = None,
 924    venv: Optional[str] = 'mrsm',
 925    split: bool = False,
 926    check_update: bool = True,
 927    check_pypi: bool = True,
 928    check_wheel: bool = True,
 929    _uninstall: bool = False,
 930    _from_completely_uninstall: bool = False,
 931    _install_uv_pip: bool = True,
 932    _use_uv_pip: bool = True,
 933    color: bool = True,
 934    silent: bool = False,
 935    dry_run: bool = False,
 936    debug: bool = False,
 937) -> bool:
 938    """
 939    Install packages from PyPI with `pip`.
 940
 941    Parameters
 942    ----------
 943    *install_names: str
 944        The installation names of packages to be installed.
 945        This includes version restrictions.
 946        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 947        from its import name.
 948
 949    args: Optional[List[str]], default None
 950        A list of command line arguments to pass to `pip`.
 951        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 952
 953    requirements_file_path: Optional[pathlib.Path, str], default None
 954        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 955
 956    venv: str, default 'mrsm'
 957        The virtual environment to install into.
 958
 959    split: bool, default False
 960        If `True`, split on periods and only install the root package name.
 961
 962    check_update: bool, default True
 963        If `True`, check if the package requires an update.
 964
 965    check_pypi: bool, default True
 966        If `True` and `check_update` is `True`, check PyPI for the latest version.
 967
 968    check_wheel: bool, default True
 969        If `True`, check if `wheel` is available.
 970
 971    _uninstall: bool, default False
 972        If `True`, uninstall packages instead.
 973
 974    color: bool, default True
 975        If `True`, include color in debug text.
 976
 977    silent: bool, default False
 978        If `True`, skip printing messages.
 979
 980    dry_run: bool, default False
 981        If `True`, print the read-only installation plan without changing the environment.
 982
 983    debug: bool, default False
 984        Verbosity toggle.
 985
 986    Returns
 987    -------
 988    A bool indicating success.
 989
 990    """
 991    import meerschaum.config.paths as paths
 992    from meerschaum._internal.static import STATIC_CONFIG
 993    from meerschaum.utils.warnings import warn
 994    if dry_run:
 995        import json
 996        plan = get_pip_install_plan(
 997            *install_names,
 998            args=args,
 999            requirements_file_path=requirements_file_path,
1000            venv=venv,
1001            _uninstall=_uninstall,
1002            _install_uv_pip=_install_uv_pip,
1003            _use_uv_pip=_use_uv_pip,
1004            debug=debug,
1005        )
1006        if not silent:
1007            print(json.dumps(plan, indent=2))
1008        return True
1009
1010    if args is None:
1011        args = ['--upgrade'] if not _uninstall else []
1012    ANSI = True if color else False
1013    if check_wheel:
1014        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
1015
1016    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
1017    inside_daemon = daemon_env_var in os.environ
1018    if inside_daemon:
1019        silent = True
1020
1021    _args = list(args)
1022    have_pip = venv_contains_package('pip', venv=None, debug=debug)
1023    pip_venv = None
1024    try:
1025        import pip
1026        have_pip = True
1027    except ImportError:
1028        have_pip = False
1029    try:
1030        import uv
1031        uv_bin = uv.find_uv_bin()
1032        have_uv_pip = True
1033    except (ImportError, FileNotFoundError):
1034        uv_bin = None
1035        have_uv_pip = False
1036
1037    if have_pip and not have_uv_pip and _install_uv_pip and is_uv_enabled():
1038        if not pip_install(
1039            'uv', 'PyYAML',
1040            venv=None,
1041            debug=debug,
1042            _install_uv_pip=False,
1043            check_update=False,
1044            check_pypi=False,
1045            check_wheel=False,
1046        ) and not silent:
1047            warn(
1048                f"Failed to install `uv` for virtual environment '{venv}'.",
1049                color=False,
1050            )
1051
1052    use_uv_pip = (
1053        _use_uv_pip
1054        and venv_contains_package('uv', venv=None, debug=debug)
1055        and uv_bin is not None
1056        and venv is not None
1057        and is_uv_enabled()
1058    )
1059
1060    import sys
1061    if not have_pip and not use_uv_pip:
1062        have_mrsm_pip = venv_contains_package('pip', venv='mrsm')
1063        if not have_mrsm_pip and not get_pip(venv=venv, color=color, debug=debug):
1064            import sys
1065            minor = sys.version_info.minor
1066            print(
1067                "\nFailed to import `pip` and `ensurepip`.\n"
1068                + "If you are running Ubuntu/Debian, "
1069                + f"you might need to install `python3.{minor}-distutils`:\n\n"
1070                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
1071                + "Please install pip and restart Meerschaum.\n\n"
1072                + "You can find instructions on installing `pip` here:\n"
1073                + "https://pip.pypa.io/en/stable/installing/"
1074            )
1075            sys.exit(1)
1076
1077        pip = attempt_import('pip', lazy=False)
1078        pip_venv = 'mrsm'
1079
1080    with Venv(venv, debug=debug):
1081        if venv is not None:
1082            if (
1083                '--ignore-installed' not in args
1084                and '-I' not in _args
1085                and not _uninstall
1086                and not use_uv_pip
1087            ):
1088                _args += ['--ignore-installed']
1089            if '--cache-dir' not in args and not _uninstall:
1090                cache_dir_path = paths.VIRTENV_RESOURCES_PATH / venv / 'cache'
1091                _args += ['--cache-dir', str(cache_dir_path)]
1092
1093        if 'pip' not in ' '.join(_args) and not use_uv_pip:
1094            if check_update and not _uninstall:
1095                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
1096                if need_update(pip, check_pypi=check_pypi, debug=debug):
1097                    _args.append(all_packages['pip'])
1098
1099        _args = (['install'] if not _uninstall else ['uninstall']) + _args
1100
1101        if check_wheel and not _uninstall and not use_uv_pip:
1102            if not have_wheel:
1103                setup_packages_to_install = (
1104                    ['setuptools', 'wheel', 'PyYAML']
1105                    + (['uv'] if is_uv_enabled() else [])
1106                )
1107                if not pip_install(
1108                    *setup_packages_to_install,
1109                    venv=venv,
1110                    check_update=False,
1111                    check_pypi=False,
1112                    check_wheel=False,
1113                    debug=debug,
1114                    _install_uv_pip=False,
1115                ) and not silent:
1116                    from meerschaum.utils.misc import items_str
1117                    warn(
1118                        (
1119                            f"Failed to install {items_str(setup_packages_to_install)} for virtual "
1120                            + f"environment '{venv}'."
1121                        ),
1122                        color=False,
1123                    )
1124
1125        if requirements_file_path is not None:
1126            _args.append('-r')
1127            _args.append(pathlib.Path(requirements_file_path).resolve().as_posix())
1128
1129        if not ANSI and '--no-color' not in _args:
1130            _args.append('--no-color')
1131
1132        if '--no-input' not in _args and not use_uv_pip:
1133            _args.append('--no-input')
1134
1135        if _uninstall and '-y' not in _args and not use_uv_pip:
1136            _args.append('-y')
1137
1138        if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip:
1139            _args.append('--no-warn-conflicts')
1140
1141        if '--disable-pip-version-check' not in _args and not use_uv_pip:
1142            _args.append('--disable-pip-version-check')
1143
1144        if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
1145            if venv is not None:
1146                vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1147                if not vtp.exists():
1148                    if not init_venv(venv, force=True):
1149                        vtp.mkdir(parents=True, exist_ok=True)
1150                _args += ['--target', venv_target_path(venv, debug=debug).as_posix()]
1151        elif (
1152            '--target' not in _args
1153                and '-t' not in _args
1154                and not inside_venv()
1155                and not _uninstall
1156                and not use_uv_pip
1157        ):
1158            _args.append('--user')
1159
1160        if venv is None and '--break-system-packages' not in _args:
1161            _args.append('--break-system-packages')
1162
1163        if debug:
1164            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
1165                if use_uv_pip:
1166                    _args.append('--verbose')
1167        else:
1168            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
1169                pass
1170
1171        _packages = [
1172            (
1173                get_install_no_version(install_name)
1174                if _uninstall or install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX)
1175                else install_name
1176            )
1177            for install_name in install_names
1178        ]
1179        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
1180        for p in _packages:
1181            msg += f'\n  - {p}'
1182        if not silent:
1183            print(msg)
1184
1185        if _uninstall and not _from_completely_uninstall and not use_uv_pip:
1186            for install_name in _packages:
1187                _install_no_version = get_install_no_version(install_name)
1188                if _install_no_version in ('pip', 'wheel', 'uv'):
1189                    continue
1190                if not completely_uninstall_package(
1191                    _install_no_version,
1192                    venv=venv,
1193                    debug=debug,
1194                ) and not silent:
1195                    warn(
1196                        f"Failed to clean up package '{_install_no_version}'.",
1197                    )
1198
1199        ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease.
1200        if use_uv_pip:
1201            _args.insert(0, 'pip')
1202            if not _uninstall and get_prerelease_dependencies(_packages):
1203                _args.append('--prerelease=allow')
1204
1205        rc = run_python_package(
1206            ('pip' if not use_uv_pip else 'uv'),
1207            _args + _packages,
1208            venv=pip_venv,
1209            env=_get_pip_os_env(color=color),
1210            debug=debug,
1211        )
1212        if debug:
1213            print(f"{rc=}")
1214        success = rc == 0
1215
1216    if success:
1217        with _locks['_is_installed_first_check']:
1218            _is_installed_first_check.clear()
1219
1220    msg = (
1221        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
1222        else "Failed to " + ('un' if _uninstall else '') + "install packages."
1223    )
1224    if not silent:
1225        print(msg)
1226    if debug and not silent:
1227        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
1228    return success
1229
1230
1231def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1232    """
1233    Return a list of explicitly prerelease dependencies from a list of packages.
1234    """
1235    if _packages is None:
1236        _packages = list(all_packages.keys())
1237    prelrease_strings = ['dev', 'rc', 'a']
1238    prerelease_packages = []
1239    for install_name in _packages:
1240        _install_no_version = get_install_no_version(install_name)
1241        import_name = _install_to_import_name(install_name)
1242        install_with_version = _import_to_install_name(import_name)
1243        version_only = (
1244            install_with_version.lower().replace(_install_no_version.lower(), '')
1245            .split(']')[-1]
1246        )
1247
1248        is_prerelease = False
1249        for prelrease_string in prelrease_strings:
1250            if prelrease_string in version_only:
1251                is_prerelease = True
1252
1253        if is_prerelease:
1254            prerelease_packages.append(install_name)
1255    return prerelease_packages
1256
1257
1258def completely_uninstall_package(
1259    install_name: str,
1260    venv: str = 'mrsm',
1261    debug: bool = False,
1262) -> bool:
1263    """
1264    Continue calling `pip uninstall` until a package is completely
1265    removed from a virtual environment. 
1266    This is useful for dealing with multiple installed versions of a package.
1267    """
1268    attempts = 0
1269    _install_no_version = get_install_no_version(install_name)
1270    clean_install_no_version = _install_no_version.lower().replace('-', '_')
1271    installed_versions = []
1272    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1273    if not vtp.exists():
1274        return True
1275
1276    for file_name in os.listdir(vtp):
1277        if not file_name.endswith('.dist-info'):
1278            continue
1279        clean_dist_info = file_name.replace('-', '_').lower()
1280        if not clean_dist_info.startswith(clean_install_no_version):
1281            continue
1282        installed_versions.append(file_name)
1283
1284    max_attempts = len(installed_versions)
1285    while attempts < max_attempts:
1286        if not venv_contains_package(
1287            _install_to_import_name(_install_no_version),
1288            venv=venv, debug=debug,
1289        ):
1290            return True
1291        if not pip_uninstall(
1292            _install_no_version,
1293            venv = venv,
1294            silent = (not debug),
1295            _from_completely_uninstall = True,
1296            debug = debug,
1297        ):
1298            return False
1299        attempts += 1
1300    return False
1301
1302
1303def pip_uninstall(
1304    *args, **kw
1305) -> bool:
1306    """
1307    Uninstall Python packages.
1308    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
1309    """
1310    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
1311
1312
1313def run_python_package(
1314    package_name: str,
1315    args: Optional[List[str]] = None,
1316    venv: Optional[str] = 'mrsm',
1317    cwd: Optional[str] = None,
1318    env: Optional[Dict[str, str]] = None,
1319    foreground: bool = False,
1320    as_proc: bool = False,
1321    capture_output: bool = False,
1322    debug: bool = False,
1323    **kw: Any,
1324) -> Union[int, subprocess.Popen, None]:
1325    """
1326    Runs an installed python package.
1327    E.g. Translates to `/usr/bin/python -m [package]`
1328
1329    Parameters
1330    ----------
1331    package_name: str
1332        The Python module to be executed.
1333
1334    args: Optional[List[str]], default None
1335        Additional command line arguments to be appended after `-m [package]`.
1336
1337    venv: Optional[str], default 'mrsm'
1338        If specified, execute the Python interpreter from a virtual environment.
1339
1340    cwd: Optional[str], default None
1341        If specified, change directories before starting the process.
1342        Defaults to `None`.
1343
1344    env: Optional[Dict[str, str]], default None
1345        If specified, only use the provided dictionary for the environment variables.
1346        Defaults to `os.environ`.
1347
1348    as_proc: bool, default False
1349        If `True`, return a `subprocess.Popen` object.
1350
1351    capture_output: bool, default False
1352        If `as_proc` is `True`, capture stdout and stderr.
1353
1354    foreground: bool, default False
1355        If `True`, start the subprocess as a foreground process.
1356        Defaults to `False`.
1357
1358    kw: Any
1359        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1360        and by extension `subprocess.Popen()`.
1361
1362    Returns
1363    -------
1364    Either a return code integer or a `subprocess.Popen` object
1365    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1366    """
1367    import sys
1368    import subprocess
1369    import traceback
1370    import meerschaum.config.paths as paths
1371    from meerschaum.utils.process import run_process
1372    from meerschaum.utils.warnings import warn
1373    if args is None:
1374        args = []
1375    old_cwd = os.getcwd()
1376    if cwd is not None:
1377        os.chdir(cwd)
1378    executable = venv_executable(venv=venv)
1379    venv_path = (paths.VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
1380    env_dict = (env if isinstance(env, dict) else (os.environ or {})).copy()
1381    if venv_path is not None:
1382        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
1383    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1384    if debug:
1385        print(command, file=sys.stderr)
1386    try:
1387        to_return = run_process(
1388            command,
1389            foreground=foreground,
1390            as_proc=as_proc,
1391            capture_output=capture_output,
1392            env=env_dict,
1393            **kw
1394        )
1395    except Exception:
1396        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1397        warn(msg, color=False)
1398        stdout, stderr = (
1399            (None, None)
1400            if not capture_output
1401            else (subprocess.PIPE, subprocess.PIPE)
1402        )
1403        proc = subprocess.Popen(
1404            command,
1405            stdout=stdout,
1406            stderr=stderr,
1407            stdin=sys.stdin,
1408            env=env_dict,
1409        )
1410        to_return = proc if as_proc else proc.wait()
1411    except KeyboardInterrupt:
1412        to_return = 1 if not as_proc else None
1413    os.chdir(old_cwd)
1414    return to_return
1415
1416
1417def attempt_import(
1418    *names: str,
1419    lazy: bool = True,
1420    warn: bool = True,
1421    install: bool = True,
1422    venv: Optional[str] = 'mrsm',
1423    precheck: bool = True,
1424    split: bool = True,
1425    check_update: bool = False,
1426    check_pypi: bool = False,
1427    check_is_installed: bool = True,
1428    allow_outside_venv: bool = True,
1429    color: bool = True,
1430    debug: bool = False
1431) -> Any:
1432    """
1433    Raise a warning if packages are not installed; otherwise import and return modules.
1434    If `lazy` is `True`, return lazy-imported modules.
1435    
1436    Returns tuple of modules if multiple names are provided, else returns one module.
1437    
1438    Parameters
1439    ----------
1440    names: List[str]
1441        The packages to be imported.
1442
1443    lazy: bool, default True
1444        If `True`, lazily load packages.
1445
1446    warn: bool, default True
1447        If `True`, raise a warning if a package cannot be imported.
1448
1449    install: bool, default True
1450        If `True`, attempt to install a missing package into the designated virtual environment.
1451        If `check_update` is True, install updates if available.
1452
1453    venv: Optional[str], default 'mrsm'
1454        The virtual environment in which to search for packages and to install packages into.
1455
1456    precheck: bool, default True
1457        If `True`, attempt to find module before importing (necessary for checking if modules exist
1458        and retaining lazy imports), otherwise assume lazy is `False`.
1459
1460    split: bool, default True
1461        If `True`, split packages' names on `'.'`.
1462
1463    check_update: bool, default False
1464        If `True` and `install` is `True`, install updates if the required minimum version
1465        does not match.
1466
1467    check_pypi: bool, default False
1468        If `True` and `check_update` is `True`, check PyPI when determining whether
1469        an update is required.
1470
1471    check_is_installed: bool, default True
1472        If `True`, check if the package is contained in the virtual environment.
1473
1474    allow_outside_venv: bool, default True
1475        If `True`, search outside of the specified virtual environment
1476        if the package cannot be found.
1477        Setting to `False` will reinstall the package into a virtual environment, even if it
1478        is installed outside.
1479
1480    color: bool, default True
1481        If `False`, do not print ANSI colors.
1482
1483    Returns
1484    -------
1485    The specified modules. If they're not available and `install` is `True`, it will first
1486    download them into a virtual environment and return the modules.
1487
1488    Examples
1489    --------
1490    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1491    >>> pandas = attempt_import('pandas')
1492
1493    """
1494
1495    import importlib.util
1496
1497    global emitted_auto_install_warning
1498    no_auto_install_env_var = 'MRSM_NO_AUTO_INSTALL'
1499    if os.environ.get(no_auto_install_env_var, '').lower() in ('1', 'true', 'yes'):
1500        install = False
1501
1502    ### to prevent recursion, check if parent Meerschaum package is being imported
1503    if names == ('meerschaum',):
1504        return _import_module('meerschaum')
1505
1506    if venv == 'mrsm' and _import_hook_venv is not None:
1507        if debug:
1508            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1509        venv = _import_hook_venv
1510
1511    _warnings = _import_module('meerschaum.utils.warnings')
1512    warn_function = _warnings.warn
1513
1514    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1515        with Venv(venv=venv, debug=debug):
1516            ### determine the import method (lazy vs normal)
1517            from meerschaum.utils.misc import filter_keywords
1518            import_method = (
1519                _import_module if not lazy
1520                else lazy_import
1521            )
1522            try:
1523                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1524            except Exception as e:
1525                if warn:
1526                    import traceback
1527                    traceback.print_exception(type(e), e, e.__traceback__)
1528                    warn_function(
1529                        f"Failed to import module '{_name}'.\nException:\n{e}",
1530                        ImportWarning,
1531                        stacklevel = (5 if lazy else 4),
1532                        color = False,
1533                    )
1534                mod = None
1535        return mod
1536
1537    modules = []
1538    for name in names:
1539        ### Check if package is a declared dependency.
1540        root_name = name.split('.')[0] if split else name
1541        install_name = _import_to_install_name(root_name)
1542
1543        if install_name is None:
1544            install_name = root_name
1545            if warn and root_name != 'plugins':
1546                warn_function(
1547                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1548                    ImportWarning,
1549                    stacklevel = 3,
1550                    color = False
1551                )
1552
1553        ### Determine if the package exists.
1554        if precheck is False:
1555            found_module = (
1556                do_import(
1557                    name, debug=debug, warn=False, venv=venv, color=color,
1558                    check_update=False, check_pypi=False, split=split,
1559                ) is not None
1560            )
1561        else:
1562            installed_cache_key = (name, venv, split, allow_outside_venv)
1563            if check_is_installed:
1564                with _locks['_is_installed_first_check']:
1565                    if not _is_installed_first_check.get(installed_cache_key, False):
1566                        package_is_installed = is_installed(
1567                            name,
1568                            venv = venv,
1569                            split = split,
1570                            allow_outside_venv = allow_outside_venv,
1571                            debug = debug,
1572                        )
1573                        if package_is_installed:
1574                            _is_installed_first_check[installed_cache_key] = True
1575                    else:
1576                        package_is_installed = True
1577            else:
1578                package_is_installed = venv_contains_package(
1579                    name,
1580                    venv=venv,
1581                    split=split,
1582                    debug=debug,
1583                )
1584            found_module = package_is_installed
1585
1586        if not found_module:
1587            if install:
1588                with _locks['emitted_auto_install_warning']:
1589                    if warn and not emitted_auto_install_warning:
1590                        emitted_auto_install_warning = True
1591                        warn_function(
1592                            "Meerschaum is installing a missing runtime dependency. "
1593                            + f"Set {no_auto_install_env_var}=1 to disable automatic downloads.",
1594                            ImportWarning,
1595                            stacklevel=3,
1596                            color=False,
1597                        )
1598                if not pip_install(
1599                    install_name,
1600                    venv = venv,
1601                    split = False,
1602                    check_update = check_update,
1603                    color = color,
1604                    debug = debug
1605                ) and warn:
1606                    warn_function(
1607                        f"Failed to install '{install_name}'.",
1608                        ImportWarning,
1609                        stacklevel = 3,
1610                        color = False,
1611                    )
1612            elif warn:
1613                ### Raise a warning if we can't find the package and install = False.
1614                warn_function(
1615                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1616                     + "some features will not work correctly."
1617                     + (
1618                         f"\n\nUnset {no_auto_install_env_var} to allow package installation.\n"
1619                         if no_auto_install_env_var in os.environ
1620                         else "\n\nSet install=True when calling attempt_import.\n"
1621                     )),
1622                    ImportWarning,
1623                    stacklevel = 3,
1624                    color = False,
1625                )
1626
1627        ### Do the import. Will be lazy if lazy=True.
1628        m = do_import(
1629            name, debug=debug, warn=warn, venv=venv, color=color,
1630            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1631        )
1632        modules.append(m)
1633
1634    modules = tuple(modules)
1635    if len(modules) == 1:
1636        return modules[0]
1637    return modules
1638
1639
1640def lazy_import(
1641    name: str,
1642    local_name: str = None,
1643    **kw
1644) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1645    """
1646    Lazily import a package.
1647    """
1648    from meerschaum.utils.packages.lazy_loader import LazyLoader
1649    if local_name is None:
1650        local_name = name
1651    return LazyLoader(
1652        local_name,
1653        globals(),
1654        name,
1655        **kw
1656    )
1657
1658
1659def pandas_name() -> str:
1660    """
1661    Return the configured name for `pandas`.
1662    
1663    Below are the expected possible values:
1664
1665    - 'pandas'
1666    - 'modin.pandas'
1667    - 'dask.dataframe'
1668
1669    """
1670    from meerschaum.config import get_config
1671    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1672    if pandas_module_name == 'modin':
1673        pandas_module_name = 'modin.pandas'
1674    elif pandas_module_name == 'dask':
1675        pandas_module_name = 'dask.dataframe'
1676
1677    return pandas_module_name
1678
1679
1680emitted_pandas_warning: bool = False
1681def import_pandas(
1682    debug: bool = False,
1683    lazy: bool = False,
1684    **kw
1685) -> 'ModuleType':
1686    """
1687    Quality-of-life function to attempt to import the configured version of `pandas`.
1688    """
1689    pandas_module_name = pandas_name()
1690    global emitted_pandas_warning
1691
1692    if pandas_module_name != 'pandas':
1693        with _locks['emitted_pandas_warning']:
1694            if not emitted_pandas_warning:
1695                from meerschaum.utils.warnings import warn
1696                emitted_pandas_warning = True
1697                warn(
1698                    (
1699                        "You are using an alternative Pandas implementation "
1700                        + f"'{pandas_module_name}'"
1701                        + "\n   Features may not work as expected."
1702                    ),
1703                    stack=False,
1704                )
1705
1706    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1707    pandas, pyarrow = attempt_import('pandas', 'pyarrow', debug=debug, lazy=False, **kw)
1708    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1709    return pd
1710
1711
1712def import_rich(
1713    lazy: bool = True,
1714    debug: bool = False,
1715    **kw: Any
1716) -> 'ModuleType':
1717    """
1718    Quality of life function for importing `rich`.
1719    """
1720    from meerschaum.utils.formatting import ANSI, UNICODE
1721    ## need typing_extensions for `from rich import box`
1722    typing_extensions = attempt_import(
1723        'typing_extensions', lazy=False, debug=debug
1724    )
1725    pygments = attempt_import(
1726        'pygments', lazy=False,
1727    )
1728    rich = attempt_import(
1729        'rich', lazy=lazy,
1730        **kw
1731    )
1732    return rich
1733
1734
1735def _dash_less_than_2(**kw) -> bool:
1736    dash = attempt_import('dash', **kw)
1737    if dash is None:
1738        return None
1739    packaging_version = attempt_import('packaging.version', **kw)
1740    return (
1741        packaging_version.parse(dash.__version__) < 
1742        packaging_version.parse('2.0.0')
1743    )
1744
1745
1746def import_dcc(warn=False, **kw) -> 'ModuleType':
1747    """
1748    Import Dash Core Components (`dcc`).
1749    """
1750    return (
1751        attempt_import('dash_core_components', warn=warn, **kw)
1752        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1753    )
1754
1755
1756def import_html(warn=False, **kw) -> 'ModuleType':
1757    """
1758    Import Dash HTML Components (`html`).
1759    """
1760    return (
1761        attempt_import('dash_html_components', warn=warn, **kw)
1762        if _dash_less_than_2(warn=warn, **kw)
1763        else attempt_import('dash.html', warn=warn, **kw)
1764    )
1765
1766
1767def get_modules_from_package(
1768    package: Any,
1769    names: bool = False,
1770    recursive: bool = False,
1771    lazy: bool = False,
1772    modules_venvs: bool = False,
1773    debug: bool = False
1774):
1775    """
1776    Find and import all modules in a package.
1777    
1778    Returns
1779    -------
1780    Either list of modules or tuple of lists.
1781    """
1782    from os.path import dirname, join, isfile, isdir, basename
1783    import glob
1784
1785    pattern = '*' if recursive else '*.py'
1786    package_path = dirname(package.__file__ or package.__path__[0])
1787    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1788    _all = [
1789        basename(f)[:-3] if isfile(f) else basename(f)
1790        for f in module_names
1791            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1792               and not f.endswith('__init__.py')
1793               and not f.endswith('__pycache__')
1794    ]
1795
1796    if debug:
1797        from meerschaum.utils.debug import dprint
1798        dprint(str(_all))
1799    modules = []
1800    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1801        ### there's probably a better way than a try: catch but it'll do for now
1802        try:
1803            ### if specified, activate the module's virtual environment before importing.
1804            ### NOTE: this only considers the filename, so two modules from different packages
1805            ### may end up sharing virtual environments.
1806            if modules_venvs:
1807                activate_venv(module_name.split('.')[-1], debug=debug)
1808            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1809            modules.append(m)
1810        except Exception:
1811            import traceback
1812            traceback.print_exc()
1813        finally:
1814            if modules_venvs:
1815                deactivate_venv(module_name.split('.')[-1], debug=debug)
1816    if names:
1817        return _all, modules
1818
1819    return modules
1820
1821
1822def import_children(
1823    package: Optional['ModuleType'] = None,
1824    package_name: Optional[str] = None,
1825    types : Optional[List[str]] = None,
1826    lazy: bool = True,
1827    recursive: bool = False,
1828    debug: bool = False
1829) -> List['ModuleType']:
1830    """
1831    Import all functions in a package to its `__init__`.
1832
1833    Parameters
1834    ----------
1835    package: Optional[ModuleType], default None
1836        Package to import its functions into.
1837        If `None` (default), use parent.
1838
1839    package_name: Optional[str], default None
1840        Name of package to import its functions into
1841        If None (default), use parent.
1842
1843    types: Optional[List[str]], default None
1844        Types of members to return.
1845        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1846
1847    Returns
1848    -------
1849    A list of modules.
1850    """
1851    import sys, inspect
1852
1853    if types is None:
1854        types = ['method', 'builtin', 'function', 'class', 'module']
1855
1856    ### if package_name and package are None, use parent
1857    if package is None and package_name is None:
1858        package_name = inspect.stack()[1][0].f_globals['__name__']
1859
1860    ### populate package or package_name from other other
1861    if package is None:
1862        package = sys.modules[package_name]
1863    elif package_name is None:
1864        package_name = package.__name__
1865
1866    ### Set attributes in sys module version of package.
1867    ### Kinda like setting a dictionary
1868    ###   functions[name] = func
1869    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1870    _all, members = [], []
1871    objects = []
1872    for module in modules:
1873        _objects = []
1874        for ob in inspect.getmembers(module):
1875            for t in types:
1876                ### ob is a tuple of (name, object)
1877                if getattr(inspect, 'is' + t)(ob[1]):
1878                    _objects.append(ob)
1879
1880        if 'module' in types:
1881            _objects.append((module.__name__.split('.')[0], module))
1882        objects += _objects
1883    for ob in objects:
1884        setattr(sys.modules[package_name], ob[0], ob[1])
1885        _all.append(ob[0])
1886        members.append(ob[1])
1887
1888    if debug:
1889        from meerschaum.utils.debug import dprint
1890        dprint(str(_all))
1891    ### set __all__ for import *
1892    setattr(sys.modules[package_name], '__all__', _all)
1893    return members
1894
1895
1896_reload_module_cache = {}
1897def reload_package(
1898    package: str,
1899    skip_submodules: Optional[List[str]] = None,
1900    lazy: bool = False,
1901    debug: bool = False,
1902    **kw: Any
1903):
1904    """
1905    Recursively load a package's subpackages, even if they were not previously loaded.
1906    """
1907    import sys
1908    if isinstance(package, str):
1909        package_name = package
1910    else:
1911        try:
1912            package_name = package.__name__
1913        except Exception as e:
1914            package_name = str(package)
1915
1916    skip_submodules = skip_submodules or []
1917    if 'meerschaum.utils.packages' not in skip_submodules:
1918        skip_submodules.append('meerschaum.utils.packages')
1919    def safeimport():
1920        subs = [
1921            m for m in sys.modules
1922            if m.startswith(package_name + '.')
1923        ]
1924        subs_to_skip = []
1925        for skip_mod in skip_submodules:
1926            for mod in subs:
1927                if mod.startswith(skip_mod):
1928                    subs_to_skip.append(mod)
1929                    continue
1930
1931        subs = [m for m in subs if m not in subs_to_skip]
1932        for module_name in subs:
1933            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1934        if not subs_to_skip:
1935            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1936
1937        return _import_module(package_name)
1938
1939    return safeimport()
1940
1941
1942def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1943    """
1944    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1945    """
1946    reload_package(
1947        'meerschaum',
1948        skip_submodules = [
1949            'meerschaum._internal.shell',
1950            'meerschaum.utils.pool',
1951        ]
1952    )
1953
1954    from meerschaum.plugins import reload_plugins
1955    from meerschaum._internal.shell.Shell import _insert_shell_actions
1956    reload_plugins(debug=debug)
1957    _insert_shell_actions()
1958    return True, "Success"
1959
1960
1961def is_installed(
1962    import_name: str,
1963    venv: Optional[str] = 'mrsm',
1964    split: bool = True,
1965    allow_outside_venv: bool = True,
1966    debug: bool = False,
1967) -> bool:
1968    """
1969    Check whether a package is installed.
1970
1971    Parameters
1972    ----------
1973    import_name: str
1974        The import name of the module.
1975
1976    venv: Optional[str], default 'mrsm'
1977        The venv in which to search for the module.
1978
1979    split: bool, default True
1980        If `True`, split on periods to determine the root module name.
1981
1982    allow_outside_venv: bool, default True
1983        If `True`, search outside of the specified virtual environment
1984        if the package cannot be found.
1985
1986    Returns
1987    -------
1988    A bool indicating whether a package may be imported.
1989    """
1990    if debug:
1991        from meerschaum.utils.debug import dprint
1992    root_name = import_name.split('.')[0] if split else import_name
1993    import importlib.util
1994    with Venv(venv, debug=debug):
1995        try:
1996            spec_path = pathlib.Path(
1997                get_module_path(root_name, venv=venv, debug=debug)
1998                or
1999                (
2000                    importlib.util.find_spec(root_name).origin 
2001                    if venv is not None and allow_outside_venv
2002                    else None
2003                )
2004            )
2005        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
2006            spec_path = None
2007
2008        found = (
2009            not need_update(
2010                None,
2011                import_name=root_name,
2012                _run_determine_version=False,
2013                check_pypi=False,
2014                version=determine_version(
2015                    spec_path,
2016                    venv=venv,
2017                    debug=debug,
2018                    import_name=root_name,
2019                ),
2020                debug=debug,
2021            )
2022        ) if spec_path is not None else False
2023
2024    return found
2025
2026
2027def venv_contains_package(
2028    import_name: str,
2029    venv: Optional[str] = 'mrsm',
2030    split: bool = True,
2031    debug: bool = False,
2032) -> bool:
2033    """
2034    Search the contents of a virtual environment for a package.
2035    """
2036    import site
2037    import pathlib
2038    root_name = import_name.split('.')[0] if split else import_name
2039    return get_module_path(root_name, venv=venv, debug=debug) is not None
2040
2041
2042def package_venv(package: 'ModuleType') -> Union[str, None]:
2043    """
2044    Inspect a package and return the virtual environment in which it presides.
2045    """
2046    import os
2047    import meerschaum.config.paths as paths
2048    if str(paths.VIRTENV_RESOURCES_PATH) not in package.__file__:
2049        return None
2050    return package.__file__.split(str(paths.VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
2051
2052
2053def ensure_readline() -> 'ModuleType':
2054    """Make sure that the `readline` package is able to be imported."""
2055    import sys
2056    try:
2057        import readline
2058    except ImportError:
2059        readline = None
2060
2061    if readline is None:
2062        import platform
2063        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
2064        try:
2065            rl = attempt_import(
2066                rl_name,
2067                lazy=False,
2068                install=True,
2069                venv=None,
2070                warn=False,
2071            )
2072        except (ImportError, ModuleNotFoundError):
2073            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
2074                print(f"Unable to import {rl_name}!", file=sys.stderr)
2075                sys.exit(1)
2076
2077    sys.modules['readline'] = readline
2078    return readline
2079
2080
2081def _get_pip_os_env(color: bool = True):
2082    """
2083    Return the environment variables context in which `pip` should be run.
2084    See PEP 668 for why we are overriding the environment.
2085    """
2086    import os, sys, platform
2087    python_bin_path = pathlib.Path(sys.executable)
2088    pip_os_env = os.environ.copy()
2089    path_str = pip_os_env.get('PATH', '') or ''
2090    path_sep = ':' if platform.system() != 'Windows' else ';'
2091    pip_os_env.update({
2092        'PIP_BREAK_SYSTEM_PACKAGES': 'true',
2093        'UV_BREAK_SYSTEM_PACKAGES': 'true',
2094        ('FORCE_COLOR' if color else 'NO_COLOR'): '1',
2095    })
2096    if str(python_bin_path) not in path_str:
2097        pip_os_env['PATH'] = str(python_bin_path.parent) + path_sep + path_str
2098
2099    return pip_os_env
2100
2101
2102def use_uv() -> bool:
2103    """
2104    Return whether `uv` is available and enabled.
2105    """
2106    from meerschaum.utils.misc import is_android
2107    if is_android():
2108        return False
2109
2110    if not is_uv_enabled():
2111        return False
2112
2113    try:
2114        import uv
2115        uv_bin = uv.find_uv_bin()
2116    except (ImportError, FileNotFoundError):
2117        uv_bin = None
2118
2119    if uv_bin is None:
2120        return False
2121
2122    return True
2123
2124
2125def is_uv_enabled() -> bool:
2126    """
2127    Return whether the user has disabled `uv`.
2128    """
2129    from meerschaum.utils.misc import is_android
2130    if is_android():
2131        return False
2132
2133    try:
2134        import yaml
2135    except ImportError:
2136        return False
2137
2138    from meerschaum.config import get_config
2139    enabled = get_config('system', 'experimental', 'uv_pip')
2140    return enabled