meerschaum.utils.packages

   1#! /usr/bin/env python
   2# -*- coding: utf-8 -*-
   3# vim:fenc=utf-8
   4
   5"""
   6Functions for managing packages and virtual environments reside here.
   7"""
   8
   9from __future__ import annotations
  10
  11import importlib.util, os, pathlib, re
  12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
  13from meerschaum.utils.threading import Lock, RLock
  14from meerschaum.utils.packages._packages import (
  15    packages,
  16    all_packages,
  17    get_install_names,
  18    _MRSM_PACKAGE_ARCHIVES_PREFIX,
  19)
  20from meerschaum.utils.venv import (
  21    activate_venv,
  22    deactivate_venv,
  23    venv_executable,
  24    venv_exec,
  25    venv_exists,
  26    venv_target_path,
  27    inside_venv,
  28    Venv,
  29    init_venv,
  30)
  31
  32_import_module = importlib.import_module
  33_import_hook_venv = None
  34_locks = {
  35    '_pkg_resources_get_distribution': RLock(),
  36    'import_versions': RLock(),
  37    '_checked_for_updates': RLock(),
  38    '_is_installed_first_check': RLock(),
  39    'emitted_pandas_warning': RLock(),
  40}
  41_checked_for_updates = set()
  42_is_installed_first_check: Dict[str, bool] = {}
  43
  44
  45def get_module_path(
  46    import_name: str,
  47    venv: Optional[str] = 'mrsm',
  48    debug: bool = False,
  49    _try_install_name_on_fail: bool = True,
  50) -> Union[pathlib.Path, None]:
  51    """
  52    Get a module's path without importing.
  53    """
  54    import site
  55    if debug:
  56        from meerschaum.utils.debug import dprint
  57    if not _try_install_name_on_fail:
  58        install_name = _import_to_install_name(import_name, with_version=False)
  59        install_name_lower = install_name.lower().replace('-', '_')
  60        import_name_lower = install_name_lower
  61    else:
  62        import_name_lower = import_name.lower().replace('-', '_')
  63
  64    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
  65    if not vtp.exists():
  66        if debug:
  67            dprint(
  68                (
  69                    "Venv '{venv}' does not exist, cannot import "
  70                    + f"'{import_name}'."
  71                ),
  72                color = False,
  73            )
  74        return None
  75
  76    venv_target_candidate_paths = [vtp]
  77    if venv is None:
  78        site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())]
  79        site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
  80
  81        paths_to_add = [
  82            path
  83            for path in site_user_packages_dirs + site_packages_dirs
  84            if path not in venv_target_candidate_paths
  85        ]
  86        venv_target_candidate_paths += paths_to_add
  87
  88    candidates = []
  89    for venv_target_candidate in venv_target_candidate_paths:
  90        try:
  91            file_names = os.listdir(venv_target_candidate)
  92        except FileNotFoundError:
  93            continue
  94        for file_name in file_names:
  95            file_name_lower = file_name.lower().replace('-', '_')
  96            if not file_name_lower.startswith(import_name_lower):
  97                continue
  98            if file_name.endswith('dist_info'):
  99                continue
 100            file_path = venv_target_candidate / file_name
 101
 102            ### Most likely: Is a directory with __init__.py
 103            if file_name_lower == import_name_lower and file_path.is_dir():
 104                init_path = file_path / '__init__.py'
 105                if init_path.exists():
 106                    candidates.append(init_path)
 107
 108            ### May be a standalone .py file.
 109            elif file_name_lower == import_name_lower + '.py':
 110                candidates.append(file_path)
 111
 112            ### Compiled wheels (e.g. pyodbc)
 113            elif file_name_lower.startswith(import_name_lower + '.'):
 114                candidates.append(file_path)
 115
 116    if len(candidates) == 1:
 117        return candidates[0]
 118
 119    if not candidates:
 120        if _try_install_name_on_fail:
 121            return get_module_path(
 122                import_name, venv=venv, debug=debug,
 123                _try_install_name_on_fail=False
 124            )
 125        return None
 126
 127    specs_paths = []
 128    for candidate_path in candidates:
 129        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
 130        if spec is not None:
 131            return candidate_path
 132    
 133    return None
 134
 135
 136def manually_import_module(
 137        import_name: str,
 138        venv: Optional[str] = 'mrsm',
 139        check_update: bool = True,
 140        check_pypi: bool = False,
 141        install: bool = True,
 142        split: bool = True,
 143        warn: bool = True,
 144        color: bool = True,
 145        debug: bool = False,
 146        use_sys_modules: bool = True,
 147    ) -> Union['ModuleType', None]:
 148    """
 149    Manually import a module from a virtual environment (or the base environment).
 150
 151    Parameters
 152    ----------
 153    import_name: str
 154        The name of the module.
 155        
 156    venv: Optional[str], default 'mrsm'
 157        The virtual environment to read from.
 158
 159    check_update: bool, default True
 160        If `True`, examine whether the available version of the package meets the required version.
 161
 162    check_pypi: bool, default False
 163        If `True`, check PyPI for updates before importing.
 164
 165    install: bool, default True
 166        If `True`, install the package if it's not installed or needs an update.
 167
 168    split: bool, default True
 169        If `True`, split `import_name` on periods to get the package name.
 170
 171    warn: bool, default True
 172        If `True`, raise a warning if the package cannot be imported.
 173
 174    color: bool, default True
 175        If `True`, use color output for debug and warning text.
 176
 177    debug: bool, default False
 178        Verbosity toggle.
 179
 180    use_sys_modules: bool, default True
 181        If `True`, return the module in `sys.modules` if it exists.
 182        Otherwise continue with manually importing.
 183
 184    Returns
 185    -------
 186    The specified module or `None` if it can't be imported.
 187
 188    """
 189    import sys
 190    _previously_imported = import_name in sys.modules
 191    if _previously_imported and use_sys_modules:
 192        return sys.modules[import_name]
 193    if debug:
 194        from meerschaum.utils.debug import dprint
 195    from meerschaum.utils.warnings import warn as warn_function
 196    import warnings
 197    root_name = import_name.split('.')[0] if split else import_name
 198    install_name = _import_to_install_name(root_name)
 199
 200    root_path = get_module_path(root_name, venv=venv)
 201    if root_path is None:
 202        return None
 203    mod_path = root_path
 204    if mod_path.is_dir():
 205        for _dir in import_name.split('.')[:-1]:
 206            mod_path = mod_path / _dir
 207            possible_end_module_filename = import_name.split('.')[-1] + '.py'
 208            try:
 209                mod_path = (
 210                    (mod_path / possible_end_module_filename)
 211                    if possible_end_module_filename in os.listdir(mod_path)
 212                    else (
 213                        mod_path / import_name.split('.')[-1] / '__init__.py'
 214                    )
 215                )
 216            except Exception as e:
 217                mod_path = None
 218
 219    spec = (
 220        importlib.util.find_spec(import_name) if mod_path is None or not mod_path.exists()
 221        else importlib.util.spec_from_file_location(import_name, str(mod_path))
 222    )
 223    root_spec = (
 224        importlib.util.find_spec(root_name) if not root_path.exists()
 225        else importlib.util.spec_from_file_location(root_name, str(root_path))
 226    )
 227
 228    ### Check for updates before importing.
 229    _version = (
 230        determine_version(
 231            pathlib.Path(root_spec.origin),
 232            import_name=root_name, venv=venv, debug=debug
 233        ) if root_spec is not None and root_spec.origin is not None else None
 234    )
 235
 236    if _version is not None:
 237        if check_update:
 238            if need_update(
 239                None,
 240                import_name=root_name,
 241                version=_version,
 242                check_pypi=check_pypi,
 243                debug=debug,
 244            ):
 245                if install:
 246                    if not pip_install(
 247                        root_name,
 248                        venv=venv,
 249                        split=False,
 250                        check_update=check_update,
 251                        color=color,
 252                        debug=debug
 253                    ) and warn:
 254                        warn_function(
 255                            f"There's an update available for '{install_name}', "
 256                            + "but it failed to install. "
 257                            + "Try installig via Meerschaum with "
 258                            + "`install packages '{install_name}'`.",
 259                            ImportWarning,
 260                            stacklevel=3,
 261                            color=False,
 262                        )
 263                elif warn:
 264                    warn_function(
 265                        f"There's an update available for '{root_name}'.",
 266                        stack=False,
 267                        color=False,
 268                    )
 269                spec = (
 270                    importlib.util.find_spec(import_name)
 271                    if mod_path is None or not mod_path.exists()
 272                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
 273                )
 274
 275    if spec is None:
 276        try:
 277            mod = _import_module(import_name)
 278        except Exception as e:
 279            mod = None
 280        return mod
 281
 282    with Venv(venv, debug=debug):
 283        mod = importlib.util.module_from_spec(spec)
 284        old_sys_mod = sys.modules.get(import_name, None)
 285        sys.modules[import_name] = mod
 286
 287        try:
 288            with warnings.catch_warnings():
 289                warnings.filterwarnings('ignore', 'The NumPy')
 290                spec.loader.exec_module(mod)
 291        except Exception as e:
 292            pass
 293        mod = _import_module(import_name)
 294        if old_sys_mod is not None:
 295            sys.modules[import_name] = old_sys_mod
 296        else:
 297            del sys.modules[import_name]
 298
 299    return mod
 300
 301
 302def _import_to_install_name(import_name: str, with_version: bool = True) -> str:
 303    """
 304    Try to translate an import name to an installation name.
 305    """
 306    install_name = all_packages.get(import_name, import_name)
 307    if with_version:
 308        return install_name
 309    return get_install_no_version(install_name)
 310
 311
 312def _import_to_dir_name(import_name: str) -> str:
 313    """
 314    Translate an import name to the package name in the sites-packages directory.
 315    """
 316    import re
 317    return re.split(
 318        r'[<>=\[]', all_packages.get(import_name, import_name)
 319    )[0].replace('-', '_').lower() 
 320
 321
 322def _install_to_import_name(install_name: str) -> str:
 323    """
 324    Translate an installation name to a package's import name.
 325    """
 326    _install_no_version = get_install_no_version(install_name)
 327    return get_install_names().get(_install_no_version, _install_no_version)
 328
 329
 330def get_install_no_version(install_name: str) -> str:
 331    """
 332    Strip the version information from the install name.
 333    """
 334    import re
 335    return re.split(r'[\[=<>,! \]]', install_name)[0]
 336
 337
 338import_versions = {}
 339def determine_version(
 340    path: pathlib.Path,
 341    import_name: Optional[str] = None,
 342    venv: Optional[str] = 'mrsm',
 343    search_for_metadata: bool = True,
 344    split: bool = True,
 345    warn: bool = False,
 346    debug: bool = False,
 347) -> Union[str, None]:
 348    """
 349    Determine a module's `__version__` string from its filepath.
 350    
 351    First it searches for pip metadata, then it attempts to import the module in a subprocess.
 352
 353    Parameters
 354    ----------
 355    path: pathlib.Path
 356        The file path of the module.
 357
 358    import_name: Optional[str], default None
 359        The name of the module. If omitted, it will be determined from the file path.
 360        Defaults to `None`.
 361
 362    venv: Optional[str], default 'mrsm'
 363        The virtual environment of the Python interpreter to use if importing is necessary.
 364
 365    search_for_metadata: bool, default True
 366        If `True`, search the pip site_packages directory (assumed to be the parent)
 367        for the corresponding dist-info directory.
 368
 369    warn: bool, default True
 370        If `True`, raise a warning if the module fails to import in the subprocess.
 371
 372    split: bool, default True
 373        If `True`, split the determined import name by periods to get the room name.
 374
 375    Returns
 376    -------
 377    The package's version string if available or `None`.
 378    If multiple versions are found, it will trigger an import in a subprocess.
 379
 380    """
 381    with _locks['import_versions']:
 382        if venv not in import_versions:
 383            import_versions[venv] = {}
 384    import os
 385    old_cwd = os.getcwd()
 386    from meerschaum.utils.warnings import warn as warn_function
 387    if import_name is None:
 388        import_name = path.parent.stem if path.stem == '__init__' else path.stem
 389        import_name = import_name.split('.')[0] if split else import_name
 390    if import_name in import_versions[venv]:
 391        return import_versions[venv][import_name]
 392    _version = None
 393    module_parent_dir = (
 394        path.parent.parent if path.stem == '__init__' else path.parent
 395    ) if path is not None else venv_target_path(venv, allow_nonexistent=True, debug=debug)
 396
 397    if not module_parent_dir.exists():
 398        return None
 399
 400    installed_dir_name = _import_to_dir_name(import_name)
 401    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
 402
 403    ### First, check if a dist-info directory exists.
 404    _found_versions = []
 405    if search_for_metadata:
 406        try:
 407            filenames = os.listdir(module_parent_dir)
 408        except FileNotFoundError:
 409            filenames = []
 410        for filename in filenames:
 411            if not filename.endswith('.dist-info'):
 412                continue
 413            filename_lower = filename.lower()
 414            if not filename_lower.startswith(clean_installed_dir_name + '-'):
 415                continue
 416            _v = filename.replace('.dist-info', '').split("-")[-1]
 417            _found_versions.append(_v)
 418
 419    if len(_found_versions) == 1:
 420        _version = _found_versions[0]
 421        with _locks['import_versions']:
 422            import_versions[venv][import_name] = _version
 423        return _found_versions[0]
 424
 425    if not _found_versions:
 426        try:
 427            import importlib.metadata as importlib_metadata
 428        except ImportError:
 429            importlib_metadata = attempt_import(
 430                'importlib_metadata',
 431                debug=debug, check_update=False, precheck=False,
 432                color=False, check_is_installed=False, lazy=False,
 433            )
 434        try:
 435            os.chdir(module_parent_dir)
 436            _version = importlib_metadata.metadata(import_name)['Version']
 437        except Exception:
 438            _version = None
 439        finally:
 440            os.chdir(old_cwd)
 441
 442        if _version is not None:
 443            with _locks['import_versions']:
 444                import_versions[venv][import_name] = _version
 445            return _version
 446
 447    if debug:
 448        print(f'Found multiple versions for {import_name}: {_found_versions}')
 449
 450    module_parent_dir_str = module_parent_dir.as_posix()
 451
 452    ### Not a pip package, so let's try importing the module directly (in a subprocess).
 453    _no_version_str = 'no-version'
 454    code = (
 455        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
 456        + f"module = importlib.import_module('{import_name}');\n"
 457        + "try:\n"
 458        + "  print(module.__version__ , end='')\n"
 459        + "except:\n"
 460        + f"  print('{_no_version_str}', end='')"
 461    )
 462    exit_code, stdout_bytes, stderr_bytes = venv_exec(
 463        code, venv=venv, with_extras=True, debug=debug
 464    )
 465    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
 466    _version = stdout.split('\n')[-1] if exit_code == 0 else None
 467    _version = _version if _version != _no_version_str else None
 468
 469    if _version is None:
 470        _version = _get_package_metadata(import_name, venv).get('version', None)
 471    if _version is None and warn:
 472        warn_function(
 473            f"Failed to determine a version for '{import_name}':\n{stderr}",
 474            stack = False
 475        )
 476
 477    ### If `__version__` doesn't exist, return `None`.
 478    import_versions[venv][import_name] = _version
 479    return _version
 480
 481
 482def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]:
 483    """
 484    Get a package's metadata from pip.
 485    This is useful for getting a version when no `__version__` is defined
 486    and multiple versions are installed.
 487
 488    Parameters
 489    ----------
 490    import_name: str
 491        The package's import or installation name.
 492
 493    venv: Optional[str]
 494        The virtual environment which contains the package.
 495
 496    Returns
 497    -------
 498    A dictionary of metadata from pip.
 499    """
 500    import re
 501    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 502    install_name = _import_to_install_name(import_name)
 503    if install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX):
 504        return {}
 505    _args = ['pip', 'show', install_name]
 506    if venv is not None:
 507        cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 508        _args += ['--cache-dir', cache_dir_path.as_posix()]
 509
 510    if use_uv():
 511        package_name = 'uv'
 512        _args = ['pip', 'show', install_name]
 513    else:
 514        package_name = 'pip'
 515        _args = ['show', install_name]
 516
 517    proc = run_python_package(
 518        package_name, _args,
 519        capture_output=True, as_proc=True, venv=venv, universal_newlines=True,
 520    )
 521    outs, errs = proc.communicate()
 522    lines = outs.split('\n')
 523    meta = {}
 524    for line in lines:
 525        vals = line.split(": ")
 526        if len(vals) != 2:
 527            continue
 528        k, v = vals[0].lower(), vals[1]
 529        if v and 'UNKNOWN' not in v:
 530            meta[k] = v
 531    return meta
 532
 533
 534def need_update(
 535    package: Optional['ModuleType'] = None,
 536    install_name: Optional[str] = None,
 537    import_name: Optional[str] = None,
 538    version: Optional[str] = None,
 539    check_pypi: bool = False,
 540    split: bool = True,
 541    color: bool = True,
 542    debug: bool = False,
 543    _run_determine_version: bool = True,
 544) -> bool:
 545    """
 546    Check if a Meerschaum dependency needs an update.
 547    Returns a bool for whether or not a package needs to be updated.
 548
 549    Parameters
 550    ----------
 551    package: 'ModuleType'
 552        The module of the package to be updated.
 553
 554    install_name: Optional[str], default None
 555        If provided, use this string to determine the required version.
 556        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
 557
 558    import_name:
 559        If provided, override the package's `__name__` string.
 560
 561    version: Optional[str], default None
 562        If specified, override the package's `__version__` string.
 563
 564    check_pypi: bool, default False
 565        If `True`, check pypi.org for updates.
 566        Defaults to `False`.
 567
 568    split: bool, default True
 569        If `True`, split the module's name on periods to detrive the root name.
 570        Defaults to `True`.
 571
 572    color: bool, default True
 573        If `True`, format debug output.
 574        Defaults to `True`.
 575
 576    debug: bool, default True
 577        Verbosity toggle.
 578
 579    Returns
 580    -------
 581    A bool indicating whether the package requires an update.
 582
 583    """
 584    if debug:
 585        from meerschaum.utils.debug import dprint
 586    from meerschaum.utils.warnings import warn as warn_function
 587    import re
 588    root_name = (
 589        package.__name__.split('.')[0] if split else package.__name__
 590    ) if import_name is None else (
 591        import_name.split('.')[0] if split else import_name
 592    )
 593    install_name = install_name or _import_to_install_name(root_name)
 594    with _locks['_checked_for_updates']:
 595        if install_name in _checked_for_updates:
 596            return False
 597        _checked_for_updates.add(install_name)
 598
 599    _install_no_version = get_install_no_version(install_name)
 600    required_version = (
 601        install_name
 602        .replace(_install_no_version, '')
 603    )
 604    if ']' in required_version:
 605        required_version = required_version.split(']')[1]
 606
 607    ### No minimum version was specified, and we're not going to check PyPI.
 608    if not required_version and not check_pypi:
 609        return False
 610
 611    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
 612    if '.dev' in required_version:
 613        required_version = required_version.split('.dev')[0]
 614    if version and '.dev' in version:
 615        version = version.split('.dev')[0]
 616
 617    try:
 618        if not version:
 619            if not _run_determine_version:
 620                version = determine_version(
 621                    pathlib.Path(package.__file__),
 622                    import_name=root_name, warn=False, debug=debug
 623                )
 624        if version is None:
 625            return False
 626    except Exception as e:
 627        if debug:
 628            dprint(str(e), color=color)
 629            dprint("No version could be determined from the installed package.", color=color)
 630        return False
 631    split_version = version.split('.')
 632    last_part = split_version[-1]
 633    if len(split_version) == 2:
 634        version = '.'.join(split_version) + '.0'
 635    elif 'dev' in last_part or 'rc' in last_part:
 636        tag = 'dev' if 'dev' in last_part else 'rc'
 637        last_sep = '-'
 638        if not last_part.startswith(tag):
 639            last_part = f'-{tag}'.join(last_part.split(tag))
 640            last_sep = '.'
 641        version = '.'.join(split_version[:-1]) + last_sep + last_part
 642    elif len(split_version) > 3:
 643        version = '.'.join(split_version[:3])
 644
 645    packaging_version = attempt_import(
 646        'packaging.version', check_update=False, lazy=False, debug=debug,
 647    )
 648
 649    ### Get semver if necessary
 650    if required_version:
 651        semver_path = get_module_path('semver', debug=debug)
 652        if semver_path is None:
 653            pip_install(_import_to_install_name('semver'), debug=debug)
 654        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
 655    if check_pypi:
 656        ### Check PyPI for updates
 657        update_checker = attempt_import(
 658            'update_checker', lazy=False, check_update=False, debug=debug
 659        )
 660        checker = update_checker.UpdateChecker()
 661        result = checker.check(_install_no_version, version)
 662    else:
 663        ### Skip PyPI and assume we can't be sure.
 664        result = None
 665
 666    ### Compare PyPI's version with our own.
 667    if result is not None:
 668        ### We have a result from PyPI and a stated required version.
 669        if required_version:
 670            try:
 671                return semver.Version.parse(result.available_version).match(required_version)
 672            except AttributeError as e:
 673                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 674                semver = manually_import_module('semver', venv='mrsm')
 675                return semver.Version.parse(version).match(required_version)
 676            except Exception as e:
 677                if debug:
 678                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
 679                return False
 680
 681        ### If `check_pypi` and we don't have a required version, check if PyPI's version
 682        ### is newer than the installed version.
 683        else:
 684            return (
 685                packaging_version.parse(result.available_version) > 
 686                packaging_version.parse(version)
 687            )
 688
 689    ### We might be depending on a prerelease.
 690    ### Sanity check that the required version is not greater than the installed version. 
 691    required_version = (
 692        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
 693        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
 694        .replace('-py3-none-any.whl', '')
 695    )
 696
 697    if 'a' in required_version:
 698        required_version = required_version.replace('a', '-pre.').replace('+mrsm', '')
 699        version = version.replace('a', '-pre.').replace('+mrsm', '')
 700    try:
 701        return (
 702            (not semver.Version.parse(version).match(required_version))
 703            if required_version else False
 704        )
 705    except AttributeError:
 706        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 707        semver = manually_import_module('semver', venv='mrsm', debug=debug)
 708        return (
 709            (not semver.Version.parse(version).match(required_version))
 710            if required_version else False
 711        )
 712    except Exception as e:
 713        print(f"Unable to parse version ({version}) for package '{import_name}'.")
 714        print(e)
 715        if debug:
 716            dprint(e)
 717        return False
 718    try:
 719        return (
 720            packaging_version.parse(version) > 
 721            packaging_version.parse(required_version)
 722        )
 723    except Exception as e:
 724        if debug:
 725            dprint(e)
 726        return False
 727    return False
 728
 729
 730def get_pip(
 731    venv: Optional[str] = 'mrsm',
 732    color: bool = True,
 733    debug: bool = False,
 734) -> bool:
 735    """
 736    Download and run the get-pip.py script.
 737
 738    Parameters
 739    ----------
 740    venv: Optional[str], default 'mrsm'
 741        The virtual environment into which to install `pip`.
 742
 743    color: bool, default True
 744        If `True`, force color output.
 745
 746    debug: bool, default False
 747        Verbosity toggle.
 748
 749    Returns
 750    -------
 751    A bool indicating success.
 752
 753    """
 754    import sys
 755    import subprocess
 756    from meerschaum.utils.misc import wget
 757    from meerschaum.config._paths import CACHE_RESOURCES_PATH
 758    from meerschaum.config.static import STATIC_CONFIG
 759    url = STATIC_CONFIG['system']['urls']['get-pip.py']
 760    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
 761    try:
 762        wget(url, dest, color=False, debug=debug)
 763    except Exception:
 764        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
 765        sys.exit(1)
 766    if venv is not None:
 767        init_venv(venv=venv, debug=debug)
 768    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
 769    return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0
 770
 771
 772def pip_install(
 773    *install_names: str,
 774    args: Optional[List[str]] = None,
 775    requirements_file_path: Union[pathlib.Path, str, None] = None,
 776    venv: Optional[str] = 'mrsm',
 777    split: bool = False,
 778    check_update: bool = True,
 779    check_pypi: bool = True,
 780    check_wheel: bool = True,
 781    _uninstall: bool = False,
 782    _from_completely_uninstall: bool = False,
 783    _install_uv_pip: bool = True,
 784    _use_uv_pip: bool = True,
 785    color: bool = True,
 786    silent: bool = False,
 787    debug: bool = False,
 788) -> bool:
 789    """
 790    Install packages from PyPI with `pip`.
 791
 792    Parameters
 793    ----------
 794    *install_names: str
 795        The installation names of packages to be installed.
 796        This includes version restrictions.
 797        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 798        from its import name.
 799
 800    args: Optional[List[str]], default None
 801        A list of command line arguments to pass to `pip`.
 802        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 803
 804    requirements_file_path: Optional[pathlib.Path, str], default None
 805        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 806
 807    venv: str, default 'mrsm'
 808        The virtual environment to install into.
 809
 810    split: bool, default False
 811        If `True`, split on periods and only install the root package name.
 812
 813    check_update: bool, default True
 814        If `True`, check if the package requires an update.
 815
 816    check_pypi: bool, default True
 817        If `True` and `check_update` is `True`, check PyPI for the latest version.
 818
 819    check_wheel: bool, default True
 820        If `True`, check if `wheel` is available.
 821
 822    _uninstall: bool, default False
 823        If `True`, uninstall packages instead.
 824
 825    color: bool, default True
 826        If `True`, include color in debug text.
 827
 828    silent: bool, default False
 829        If `True`, skip printing messages.
 830
 831    debug: bool, default False
 832        Verbosity toggle.
 833
 834    Returns
 835    -------
 836    A bool indicating success.
 837
 838    """
 839    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 840    from meerschaum.config.static import STATIC_CONFIG
 841    from meerschaum.utils.warnings import warn
 842    if args is None:
 843        args = ['--upgrade'] if not _uninstall else []
 844    ANSI = True if color else False
 845    if check_wheel:
 846        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 847
 848    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
 849    inside_daemon = daemon_env_var in os.environ
 850    if inside_daemon:
 851        silent = True
 852
 853    _args = list(args)
 854    have_pip = venv_contains_package('pip', venv=None, debug=debug)
 855    try:
 856        import pip
 857        have_pip = True
 858    except ImportError:
 859        have_pip = False
 860    try:
 861        import uv
 862        uv_bin = uv.find_uv_bin()
 863        have_uv_pip = True
 864    except (ImportError, FileNotFoundError):
 865        uv_bin = None
 866        have_uv_pip = False
 867    if have_pip and not have_uv_pip and _install_uv_pip and is_uv_enabled():
 868        if not pip_install(
 869            'uv',
 870            venv=None,
 871            debug=debug,
 872            _install_uv_pip=False,
 873            check_update=False,
 874            check_pypi=False,
 875            check_wheel=False,
 876        ) and not silent:
 877            warn(
 878                f"Failed to install `uv` for virtual environment '{venv}'.",
 879                color=False,
 880            )
 881
 882    use_uv_pip = (
 883        _use_uv_pip
 884        and venv_contains_package('uv', venv=None, debug=debug)
 885        and uv_bin is not None
 886        and venv is not None
 887        and is_uv_enabled()
 888    )
 889
 890    import sys
 891    if not have_pip and not use_uv_pip:
 892        if not get_pip(venv=venv, color=color, debug=debug):
 893            import sys
 894            minor = sys.version_info.minor
 895            print(
 896                "\nFailed to import `pip` and `ensurepip`.\n"
 897                + "If you are running Ubuntu/Debian, "
 898                + "you might need to install `python3.{minor}-distutils`:\n\n"
 899                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
 900                + "Please install pip and restart Meerschaum.\n\n"
 901                + "You can find instructions on installing `pip` here:\n"
 902                + "https://pip.pypa.io/en/stable/installing/"
 903            )
 904            sys.exit(1)
 905
 906    with Venv(venv, debug=debug):
 907        if venv is not None:
 908            if (
 909                '--ignore-installed' not in args
 910                and '-I' not in _args
 911                and not _uninstall
 912                and not use_uv_pip
 913            ):
 914                _args += ['--ignore-installed']
 915            if '--cache-dir' not in args and not _uninstall:
 916                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 917                _args += ['--cache-dir', str(cache_dir_path)]
 918
 919        if 'pip' not in ' '.join(_args) and not use_uv_pip:
 920            if check_update and not _uninstall:
 921                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
 922                if need_update(pip, check_pypi=check_pypi, debug=debug):
 923                    _args.append(all_packages['pip'])
 924
 925        _args = (['install'] if not _uninstall else ['uninstall']) + _args
 926
 927        if check_wheel and not _uninstall and not use_uv_pip:
 928            if not have_wheel:
 929                setup_packages_to_install = (
 930                    ['setuptools', 'wheel']
 931                    + (['uv'] if is_uv_enabled() else [])
 932                )
 933                if not pip_install(
 934                    *setup_packages_to_install,
 935                    venv=venv,
 936                    check_update=False,
 937                    check_pypi=False,
 938                    check_wheel=False,
 939                    debug=debug,
 940                    _install_uv_pip=False,
 941                ) and not silent:
 942                    from meerschaum.utils.misc import items_str
 943                    warn(
 944                        (
 945                            f"Failed to install {items_str(setup_packages_to_install)} for virtual "
 946                            + f"environment '{venv}'."
 947                        ),
 948                        color=False,
 949                    )
 950
 951        if requirements_file_path is not None:
 952            _args.append('-r')
 953            _args.append(pathlib.Path(requirements_file_path).resolve().as_posix())
 954
 955        if not ANSI and '--no-color' not in _args:
 956            _args.append('--no-color')
 957
 958        if '--no-input' not in _args and not use_uv_pip:
 959            _args.append('--no-input')
 960
 961        if _uninstall and '-y' not in _args and not use_uv_pip:
 962            _args.append('-y')
 963
 964        if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip:
 965            _args.append('--no-warn-conflicts')
 966
 967        if '--disable-pip-version-check' not in _args and not use_uv_pip:
 968            _args.append('--disable-pip-version-check')
 969
 970        if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
 971            if venv is not None:
 972                vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 973                if not vtp.exists():
 974                    if not init_venv(venv, force=True):
 975                        vtp.mkdir(parents=True, exist_ok=True)
 976                _args += ['--target', venv_target_path(venv, debug=debug)]
 977        elif (
 978            '--target' not in _args
 979                and '-t' not in _args
 980                and not inside_venv()
 981                and not _uninstall
 982                and not use_uv_pip
 983        ):
 984            _args += ['--user']
 985
 986        if debug:
 987            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
 988                if use_uv_pip:
 989                    _args.append('--verbose')
 990        else:
 991            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
 992                pass
 993
 994        _packages = [
 995            (
 996                get_install_no_version(install_name)
 997                if _uninstall or install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX)
 998                else install_name
 999            )
1000            for install_name in install_names
1001        ]
1002        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
1003        for p in _packages:
1004            msg += f'\n  - {p}'
1005        if not silent:
1006            print(msg)
1007
1008        if _uninstall and not _from_completely_uninstall and not use_uv_pip:
1009            for install_name in _packages:
1010                _install_no_version = get_install_no_version(install_name)
1011                if _install_no_version in ('pip', 'wheel', 'uv'):
1012                    continue
1013                if not completely_uninstall_package(
1014                    _install_no_version,
1015                    venv=venv, debug=debug,
1016                ) and not silent:
1017                    warn(
1018                        f"Failed to clean up package '{_install_no_version}'.",
1019                    )
1020
1021        ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease.
1022        if use_uv_pip:
1023            _args.insert(0, 'pip')
1024            if not _uninstall and get_prerelease_dependencies(_packages):
1025                _args.append('--prerelease=allow')
1026
1027        rc = run_python_package(
1028            ('pip' if not use_uv_pip else 'uv'),
1029            _args + _packages,
1030            venv=None,
1031            env=_get_pip_os_env(color=color),
1032            debug=debug,
1033        )
1034        if debug:
1035            print(f"{rc=}")
1036        success = rc == 0
1037
1038    msg = (
1039        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
1040        else "Failed to " + ('un' if _uninstall else '') + "install packages."
1041    )
1042    if not silent:
1043        print(msg)
1044    if debug and not silent:
1045        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
1046    return success
1047
1048
1049def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1050    """
1051    Return a list of explicitly prerelease dependencies from a list of packages.
1052    """
1053    if _packages is None:
1054        _packages = list(all_packages.keys())
1055    prelrease_strings = ['dev', 'rc', 'a']
1056    prerelease_packages = []
1057    for install_name in _packages:
1058        _install_no_version = get_install_no_version(install_name)
1059        import_name = _install_to_import_name(install_name)
1060        install_with_version = _import_to_install_name(import_name)
1061        version_only = (
1062            install_with_version.lower().replace(_install_no_version.lower(), '')
1063            .split(']')[-1]
1064        )
1065
1066        is_prerelease = False
1067        for prelrease_string in prelrease_strings:
1068            if prelrease_string in version_only:
1069                is_prerelease = True
1070
1071        if is_prerelease:
1072            prerelease_packages.append(install_name)
1073    return prerelease_packages
1074
1075
1076def completely_uninstall_package(
1077    install_name: str,
1078    venv: str = 'mrsm',
1079    debug: bool = False,
1080) -> bool:
1081    """
1082    Continue calling `pip uninstall` until a package is completely
1083    removed from a virtual environment. 
1084    This is useful for dealing with multiple installed versions of a package.
1085    """
1086    attempts = 0
1087    _install_no_version = get_install_no_version(install_name)
1088    clean_install_no_version = _install_no_version.lower().replace('-', '_')
1089    installed_versions = []
1090    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1091    if not vtp.exists():
1092        return True
1093
1094    for file_name in os.listdir(vtp):
1095        if not file_name.endswith('.dist-info'):
1096            continue
1097        clean_dist_info = file_name.replace('-', '_').lower()
1098        if not clean_dist_info.startswith(clean_install_no_version):
1099            continue
1100        installed_versions.append(file_name)
1101
1102    max_attempts = len(installed_versions)
1103    while attempts < max_attempts:
1104        if not venv_contains_package(
1105            _install_to_import_name(_install_no_version),
1106            venv=venv, debug=debug,
1107        ):
1108            return True
1109        if not pip_uninstall(
1110            _install_no_version,
1111            venv = venv,
1112            silent = (not debug),
1113            _from_completely_uninstall = True,
1114            debug = debug,
1115        ):
1116            return False
1117        attempts += 1
1118    return False
1119
1120
1121def pip_uninstall(
1122    *args, **kw
1123) -> bool:
1124    """
1125    Uninstall Python packages.
1126    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
1127    """
1128    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
1129
1130
1131def run_python_package(
1132    package_name: str,
1133    args: Optional[List[str]] = None,
1134    venv: Optional[str] = 'mrsm',
1135    cwd: Optional[str] = None,
1136    foreground: bool = False,
1137    as_proc: bool = False,
1138    capture_output: bool = False,
1139    debug: bool = False,
1140    **kw: Any,
1141) -> Union[int, subprocess.Popen, None]:
1142    """
1143    Runs an installed python package.
1144    E.g. Translates to `/usr/bin/python -m [package]`
1145
1146    Parameters
1147    ----------
1148    package_name: str
1149        The Python module to be executed.
1150
1151    args: Optional[List[str]], default None
1152        Additional command line arguments to be appended after `-m [package]`.
1153
1154    venv: Optional[str], default 'mrsm'
1155        If specified, execute the Python interpreter from a virtual environment.
1156
1157    cwd: Optional[str], default None
1158        If specified, change directories before starting the process.
1159        Defaults to `None`.
1160
1161    as_proc: bool, default False
1162        If `True`, return a `subprocess.Popen` object.
1163
1164    capture_output: bool, default False
1165        If `as_proc` is `True`, capture stdout and stderr.
1166
1167    foreground: bool, default False
1168        If `True`, start the subprocess as a foreground process.
1169        Defaults to `False`.
1170
1171    kw: Any
1172        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1173        and by extension `subprocess.Popen()`.
1174
1175    Returns
1176    -------
1177    Either a return code integer or a `subprocess.Popen` object
1178    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1179    """
1180    import sys, platform
1181    import subprocess
1182    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1183    from meerschaum.utils.process import run_process
1184    from meerschaum.utils.warnings import warn
1185    if args is None:
1186        args = []
1187    old_cwd = os.getcwd()
1188    if cwd is not None:
1189        os.chdir(cwd)
1190    executable = venv_executable(venv=venv)
1191    venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
1192    env_dict = kw.get('env', os.environ).copy()
1193    if venv_path is not None:
1194        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
1195    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1196    import traceback
1197    if debug:
1198        print(command, file=sys.stderr)
1199    try:
1200        to_return = run_process(
1201            command,
1202            foreground=foreground,
1203            as_proc=as_proc,
1204            capture_output=capture_output,
1205            **kw
1206        )
1207    except Exception as e:
1208        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1209        warn(msg, color=False)
1210        stdout, stderr = (
1211            (None, None)
1212            if not capture_output
1213            else (subprocess.PIPE, subprocess.PIPE)
1214        )
1215        proc = subprocess.Popen(
1216            command,
1217            stdout=stdout,
1218            stderr=stderr,
1219            env=env_dict,
1220        )
1221        to_return = proc if as_proc else proc.wait()
1222    except KeyboardInterrupt:
1223        to_return = 1 if not as_proc else None
1224    os.chdir(old_cwd)
1225    return to_return
1226
1227
1228def attempt_import(
1229    *names: str,
1230    lazy: bool = True,
1231    warn: bool = True,
1232    install: bool = True,
1233    venv: Optional[str] = 'mrsm',
1234    precheck: bool = True,
1235    split: bool = True,
1236    check_update: bool = False,
1237    check_pypi: bool = False,
1238    check_is_installed: bool = True,
1239    allow_outside_venv: bool = True,
1240    color: bool = True,
1241    debug: bool = False
1242) -> Any:
1243    """
1244    Raise a warning if packages are not installed; otherwise import and return modules.
1245    If `lazy` is `True`, return lazy-imported modules.
1246    
1247    Returns tuple of modules if multiple names are provided, else returns one module.
1248    
1249    Parameters
1250    ----------
1251    names: List[str]
1252        The packages to be imported.
1253
1254    lazy: bool, default True
1255        If `True`, lazily load packages.
1256
1257    warn: bool, default True
1258        If `True`, raise a warning if a package cannot be imported.
1259
1260    install: bool, default True
1261        If `True`, attempt to install a missing package into the designated virtual environment.
1262        If `check_update` is True, install updates if available.
1263
1264    venv: Optional[str], default 'mrsm'
1265        The virtual environment in which to search for packages and to install packages into.
1266
1267    precheck: bool, default True
1268        If `True`, attempt to find module before importing (necessary for checking if modules exist
1269        and retaining lazy imports), otherwise assume lazy is `False`.
1270
1271    split: bool, default True
1272        If `True`, split packages' names on `'.'`.
1273
1274    check_update: bool, default False
1275        If `True` and `install` is `True`, install updates if the required minimum version
1276        does not match.
1277
1278    check_pypi: bool, default False
1279        If `True` and `check_update` is `True`, check PyPI when determining whether
1280        an update is required.
1281
1282    check_is_installed: bool, default True
1283        If `True`, check if the package is contained in the virtual environment.
1284
1285    allow_outside_venv: bool, default True
1286        If `True`, search outside of the specified virtual environment
1287        if the package cannot be found.
1288        Setting to `False` will reinstall the package into a virtual environment, even if it
1289        is installed outside.
1290
1291    color: bool, default True
1292        If `False`, do not print ANSI colors.
1293
1294    Returns
1295    -------
1296    The specified modules. If they're not available and `install` is `True`, it will first
1297    download them into a virtual environment and return the modules.
1298
1299    Examples
1300    --------
1301    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1302    >>> pandas = attempt_import('pandas')
1303
1304    """
1305
1306    import importlib.util
1307
1308    ### to prevent recursion, check if parent Meerschaum package is being imported
1309    if names == ('meerschaum',):
1310        return _import_module('meerschaum')
1311
1312    if venv == 'mrsm' and _import_hook_venv is not None:
1313        if debug:
1314            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1315        venv = _import_hook_venv
1316
1317    _warnings = _import_module('meerschaum.utils.warnings')
1318    warn_function = _warnings.warn
1319
1320    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1321        with Venv(venv=venv, debug=debug):
1322            ### determine the import method (lazy vs normal)
1323            from meerschaum.utils.misc import filter_keywords
1324            import_method = (
1325                _import_module if not lazy
1326                else lazy_import
1327            )
1328            try:
1329                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1330            except Exception as e:
1331                if warn:
1332                    import traceback
1333                    traceback.print_exception(type(e), e, e.__traceback__)
1334                    warn_function(
1335                        f"Failed to import module '{_name}'.\nException:\n{e}",
1336                        ImportWarning,
1337                        stacklevel = (5 if lazy else 4),
1338                        color = False,
1339                    )
1340                mod = None
1341        return mod
1342
1343    modules = []
1344    for name in names:
1345        ### Check if package is a declared dependency.
1346        root_name = name.split('.')[0] if split else name
1347        install_name = _import_to_install_name(root_name)
1348
1349        if install_name is None:
1350            install_name = root_name
1351            if warn and root_name != 'plugins':
1352                warn_function(
1353                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1354                    ImportWarning,
1355                    stacklevel = 3,
1356                    color = False
1357                )
1358
1359        ### Determine if the package exists.
1360        if precheck is False:
1361            found_module = (
1362                do_import(
1363                    name, debug=debug, warn=False, venv=venv, color=color,
1364                    check_update=False, check_pypi=False, split=split,
1365                ) is not None
1366            )
1367        else:
1368            if check_is_installed:
1369                with _locks['_is_installed_first_check']:
1370                    if not _is_installed_first_check.get(name, False):
1371                        package_is_installed = is_installed(
1372                            name,
1373                            venv = venv,
1374                            split = split,
1375                            allow_outside_venv = allow_outside_venv,
1376                            debug = debug,
1377                        )
1378                        _is_installed_first_check[name] = package_is_installed
1379                    else:
1380                        package_is_installed = _is_installed_first_check[name]
1381            else:
1382                package_is_installed = _is_installed_first_check.get(
1383                    name,
1384                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1385                )
1386            found_module = package_is_installed
1387
1388        if not found_module:
1389            if install:
1390                if not pip_install(
1391                    install_name,
1392                    venv = venv,
1393                    split = False,
1394                    check_update = check_update,
1395                    color = color,
1396                    debug = debug
1397                ) and warn:
1398                    warn_function(
1399                        f"Failed to install '{install_name}'.",
1400                        ImportWarning,
1401                        stacklevel = 3,
1402                        color = False,
1403                    )
1404            elif warn:
1405                ### Raise a warning if we can't find the package and install = False.
1406                warn_function(
1407                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1408                     + "some features will not work correctly."
1409                     + f"\n\nSet install=True when calling attempt_import.\n"),
1410                    ImportWarning,
1411                    stacklevel = 3,
1412                    color = False,
1413                )
1414
1415        ### Do the import. Will be lazy if lazy=True.
1416        m = do_import(
1417            name, debug=debug, warn=warn, venv=venv, color=color,
1418            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1419        )
1420        modules.append(m)
1421
1422    modules = tuple(modules)
1423    if len(modules) == 1:
1424        return modules[0]
1425    return modules
1426
1427
1428def lazy_import(
1429    name: str,
1430    local_name: str = None,
1431    **kw
1432) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1433    """
1434    Lazily import a package.
1435    """
1436    from meerschaum.utils.packages.lazy_loader import LazyLoader
1437    if local_name is None:
1438        local_name = name
1439    return LazyLoader(
1440        local_name,
1441        globals(),
1442        name,
1443        **kw
1444    )
1445
1446
1447def pandas_name() -> str:
1448    """
1449    Return the configured name for `pandas`.
1450    
1451    Below are the expected possible values:
1452
1453    - 'pandas'
1454    - 'modin.pandas'
1455    - 'dask.dataframe'
1456
1457    """
1458    from meerschaum.config import get_config
1459    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1460    if pandas_module_name == 'modin':
1461        pandas_module_name = 'modin.pandas'
1462    elif pandas_module_name == 'dask':
1463        pandas_module_name = 'dask.dataframe'
1464
1465    return pandas_module_name
1466
1467
1468emitted_pandas_warning: bool = False
1469def import_pandas(
1470    debug: bool = False,
1471    lazy: bool = False,
1472    **kw
1473) -> 'ModuleType':
1474    """
1475    Quality-of-life function to attempt to import the configured version of `pandas`.
1476    """
1477    pandas_module_name = pandas_name()
1478    global emitted_pandas_warning
1479
1480    if pandas_module_name != 'pandas':
1481        with _locks['emitted_pandas_warning']:
1482            if not emitted_pandas_warning:
1483                from meerschaum.utils.warnings import warn
1484                emitted_pandas_warning = True
1485                warn(
1486                    (
1487                        "You are using an alternative Pandas implementation "
1488                        + f"'{pandas_module_name}'"
1489                        + "\n   Features may not work as expected."
1490                    ),
1491                    stack=False,
1492                )
1493
1494    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1495    pandas, pyarrow = attempt_import('pandas', 'pyarrow', debug=debug, lazy=False, **kw)
1496    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1497    return pd
1498
1499
1500def import_rich(
1501    lazy: bool = True,
1502    debug: bool = False,
1503    **kw : Any
1504) -> 'ModuleType':
1505    """
1506    Quality of life function for importing `rich`.
1507    """
1508    from meerschaum.utils.formatting import ANSI, UNICODE
1509    if not ANSI and not UNICODE:
1510        return None
1511
1512    ## need typing_extensions for `from rich import box`
1513    typing_extensions = attempt_import(
1514        'typing_extensions', lazy=False, debug=debug
1515    )
1516    pygments = attempt_import(
1517        'pygments', lazy=False,
1518    )
1519    rich = attempt_import(
1520        'rich', lazy=lazy,
1521        **kw
1522    )
1523    return rich
1524
1525
1526def _dash_less_than_2(**kw) -> bool:
1527    dash = attempt_import('dash', **kw)
1528    if dash is None:
1529        return None
1530    packaging_version = attempt_import('packaging.version', **kw)
1531    return (
1532        packaging_version.parse(dash.__version__) < 
1533        packaging_version.parse('2.0.0')
1534    )
1535
1536
1537def import_dcc(warn=False, **kw) -> 'ModuleType':
1538    """
1539    Import Dash Core Components (`dcc`).
1540    """
1541    return (
1542        attempt_import('dash_core_components', warn=warn, **kw)
1543        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1544    )
1545
1546
1547def import_html(warn=False, **kw) -> 'ModuleType':
1548    """
1549    Import Dash HTML Components (`html`).
1550    """
1551    return (
1552        attempt_import('dash_html_components', warn=warn, **kw)
1553        if _dash_less_than_2(warn=warn, **kw)
1554        else attempt_import('dash.html', warn=warn, **kw)
1555    )
1556
1557
1558def get_modules_from_package(
1559    package: 'package',
1560    names: bool = False,
1561    recursive: bool = False,
1562    lazy: bool = False,
1563    modules_venvs: bool = False,
1564    debug: bool = False
1565):
1566    """
1567    Find and import all modules in a package.
1568    
1569    Returns
1570    -------
1571    Either list of modules or tuple of lists.
1572    """
1573    from os.path import dirname, join, isfile, isdir, basename
1574    import glob
1575
1576    pattern = '*' if recursive else '*.py'
1577    package_path = dirname(package.__file__ or package.__path__[0])
1578    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1579    _all = [
1580        basename(f)[:-3] if isfile(f) else basename(f)
1581        for f in module_names
1582            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1583               and not f.endswith('__init__.py')
1584               and not f.endswith('__pycache__')
1585    ]
1586
1587    if debug:
1588        from meerschaum.utils.debug import dprint
1589        dprint(str(_all))
1590    modules = []
1591    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1592        ### there's probably a better way than a try: catch but it'll do for now
1593        try:
1594            ### if specified, activate the module's virtual environment before importing.
1595            ### NOTE: this only considers the filename, so two modules from different packages
1596            ### may end up sharing virtual environments.
1597            if modules_venvs:
1598                activate_venv(module_name.split('.')[-1], debug=debug)
1599            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1600            modules.append(m)
1601        except Exception as e:
1602            if debug:
1603                dprint(str(e))
1604        finally:
1605            if modules_venvs:
1606                deactivate_venv(module_name.split('.')[-1], debug=debug)
1607    if names:
1608        return _all, modules
1609
1610    return modules
1611
1612
1613def import_children(
1614    package: Optional['ModuleType'] = None,
1615    package_name: Optional[str] = None,
1616    types : Optional[List[str]] = None,
1617    lazy: bool = True,
1618    recursive: bool = False,
1619    debug: bool = False
1620) -> List['ModuleType']:
1621    """
1622    Import all functions in a package to its `__init__`.
1623
1624    Parameters
1625    ----------
1626    package: Optional[ModuleType], default None
1627        Package to import its functions into.
1628        If `None` (default), use parent.
1629
1630    package_name: Optional[str], default None
1631        Name of package to import its functions into
1632        If None (default), use parent.
1633
1634    types: Optional[List[str]], default None
1635        Types of members to return.
1636        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1637
1638    Returns
1639    -------
1640    A list of modules.
1641    """
1642    import sys, inspect
1643
1644    if types is None:
1645        types = ['method', 'builtin', 'function', 'class', 'module']
1646
1647    ### if package_name and package are None, use parent
1648    if package is None and package_name is None:
1649        package_name = inspect.stack()[1][0].f_globals['__name__']
1650
1651    ### populate package or package_name from other other
1652    if package is None:
1653        package = sys.modules[package_name]
1654    elif package_name is None:
1655        package_name = package.__name__
1656
1657    ### Set attributes in sys module version of package.
1658    ### Kinda like setting a dictionary
1659    ###   functions[name] = func
1660    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1661    _all, members = [], []
1662    objects = []
1663    for module in modules:
1664        _objects = []
1665        for ob in inspect.getmembers(module):
1666            for t in types:
1667                ### ob is a tuple of (name, object)
1668                if getattr(inspect, 'is' + t)(ob[1]):
1669                    _objects.append(ob)
1670
1671        if 'module' in types:
1672            _objects.append((module.__name__.split('.')[0], module))
1673        objects += _objects
1674    for ob in objects:
1675        setattr(sys.modules[package_name], ob[0], ob[1])
1676        _all.append(ob[0])
1677        members.append(ob[1])
1678
1679    if debug:
1680        from meerschaum.utils.debug import dprint
1681        dprint(str(_all))
1682    ### set __all__ for import *
1683    setattr(sys.modules[package_name], '__all__', _all)
1684    return members
1685
1686
1687_reload_module_cache = {}
1688def reload_package(
1689    package: str,
1690    skip_submodules: Optional[List[str]] = None,
1691    lazy: bool = False,
1692    debug: bool = False,
1693    **kw: Any
1694):
1695    """
1696    Recursively load a package's subpackages, even if they were not previously loaded.
1697    """
1698    import sys
1699    if isinstance(package, str):
1700        package_name = package
1701    else:
1702        try:
1703            package_name = package.__name__
1704        except Exception as e:
1705            package_name = str(package)
1706
1707    skip_submodules = skip_submodules or []
1708    if 'meerschaum.utils.packages' not in skip_submodules:
1709        skip_submodules.append('meerschaum.utils.packages')
1710    def safeimport():
1711        subs = [
1712            m for m in sys.modules
1713            if m.startswith(package_name + '.')
1714        ]
1715        subs_to_skip = []
1716        for skip_mod in skip_submodules:
1717            for mod in subs:
1718                if mod.startswith(skip_mod):
1719                    subs_to_skip.append(mod)
1720                    continue
1721
1722        subs = [m for m in subs if m not in subs_to_skip]
1723        for module_name in subs:
1724            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1725        if not subs_to_skip:
1726            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1727
1728        return _import_module(package_name)
1729
1730    return safeimport()
1731
1732
1733def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1734    """
1735    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1736    """
1737    reload_package(
1738        'meerschaum',
1739        skip_submodules = [
1740            'meerschaum._internal.shell',
1741            'meerschaum.utils.pool',
1742        ]
1743    )
1744
1745    from meerschaum.plugins import reload_plugins
1746    from meerschaum._internal.shell.Shell import _insert_shell_actions
1747    reload_plugins(debug=debug)
1748    _insert_shell_actions()
1749    return True, "Success"
1750
1751
1752def is_installed(
1753    import_name: str,
1754    venv: Optional[str] = 'mrsm',
1755    split: bool = True,
1756    allow_outside_venv: bool = True,
1757    debug: bool = False,
1758) -> bool:
1759    """
1760    Check whether a package is installed.
1761
1762    Parameters
1763    ----------
1764    import_name: str
1765        The import name of the module.
1766
1767    venv: Optional[str], default 'mrsm'
1768        The venv in which to search for the module.
1769
1770    split: bool, default True
1771        If `True`, split on periods to determine the root module name.
1772
1773    allow_outside_venv: bool, default True
1774        If `True`, search outside of the specified virtual environment
1775        if the package cannot be found.
1776    """
1777    if debug:
1778        from meerschaum.utils.debug import dprint
1779    root_name = import_name.split('.')[0] if split else import_name
1780    import importlib.util
1781    with Venv(venv, debug=debug):
1782        try:
1783            spec_path = pathlib.Path(
1784                get_module_path(root_name, venv=venv, debug=debug)
1785                or
1786                (
1787                    importlib.util.find_spec(root_name).origin 
1788                    if venv is not None and allow_outside_venv
1789                    else None
1790                )
1791            )
1792        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1793            spec_path = None
1794
1795        found = (
1796            not need_update(
1797                None,
1798                import_name=root_name,
1799                _run_determine_version=False,
1800                check_pypi=False,
1801                version=determine_version(
1802                    spec_path,
1803                    venv=venv,
1804                    debug=debug,
1805                    import_name=root_name,
1806                ),
1807                debug=debug,
1808            )
1809        ) if spec_path is not None else False
1810
1811    return found
1812
1813
1814def venv_contains_package(
1815    import_name: str,
1816    venv: Optional[str] = 'mrsm',
1817    split: bool = True,
1818    debug: bool = False,
1819) -> bool:
1820    """
1821    Search the contents of a virtual environment for a package.
1822    """
1823    import site
1824    import pathlib
1825    root_name = import_name.split('.')[0] if split else import_name
1826    return get_module_path(root_name, venv=venv, debug=debug) is not None
1827
1828
1829def package_venv(package: 'ModuleType') -> Union[str, None]:
1830    """
1831    Inspect a package and return the virtual environment in which it presides.
1832    """
1833    import os
1834    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1835    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1836        return None
1837    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
1838
1839
1840def ensure_readline() -> 'ModuleType':
1841    """Make sure that the `readline` package is able to be imported."""
1842    import sys
1843    try:
1844        import readline
1845    except ImportError:
1846        readline = None
1847
1848    if readline is None:
1849        import platform
1850        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1851        try:
1852            rl = attempt_import(
1853                rl_name,
1854                lazy=False,
1855                install=True,
1856                venv=None,
1857                warn=False,
1858            )
1859        except (ImportError, ModuleNotFoundError):
1860            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1861                print(f"Unable to import {rl_name}!", file=sys.stderr)
1862                sys.exit(1)
1863
1864    sys.modules['readline'] = readline
1865    return readline
1866
1867_pkg_resources_get_distribution = None
1868_custom_distributions = {}
1869def _monkey_patch_get_distribution(_dist: str, _version: str) -> None:
1870    """
1871    Monkey patch `pkg_resources.get_distribution` to allow for importing `flask_compress`.
1872    """
1873    import pkg_resources
1874    from collections import namedtuple
1875    global _pkg_resources_get_distribution
1876    with _locks['_pkg_resources_get_distribution']:
1877        _pkg_resources_get_distribution = pkg_resources.get_distribution
1878    _custom_distributions[_dist] = _version
1879    _Dist = namedtuple('_Dist', ['version'])
1880    def _get_distribution(dist):
1881        """Hack for flask-compress."""
1882        if dist in _custom_distributions:
1883            return _Dist(_custom_distributions[dist])
1884        return _pkg_resources_get_distribution(dist)
1885    pkg_resources.get_distribution = _get_distribution
1886
1887
1888def _get_pip_os_env(color: bool = True):
1889    """
1890    Return the environment variables context in which `pip` should be run.
1891    See PEP 668 for why we are overriding the environment.
1892    """
1893    import os, sys, platform
1894    python_bin_path = pathlib.Path(sys.executable)
1895    pip_os_env = os.environ.copy()
1896    path_str = pip_os_env.get('PATH', '') or ''
1897    path_sep = ':' if platform.system() != 'Windows' else ';'
1898    pip_os_env.update({
1899        'PIP_BREAK_SYSTEM_PACKAGES': 'true',
1900        'UV_BREAK_SYSTEM_PACKAGES': 'true',
1901        ('FORCE_COLOR' if color else 'NO_COLOR'): '1',
1902    })
1903    if str(python_bin_path) not in path_str:
1904        pip_os_env['PATH'] = str(python_bin_path.parent) + path_sep + path_str
1905
1906    return pip_os_env
1907
1908
1909def use_uv() -> bool:
1910    """
1911    Return whether `uv` is available and enabled.
1912    """
1913    from meerschaum.utils.misc import is_android
1914    if is_android():
1915        return False
1916
1917    if not is_uv_enabled():
1918        return False
1919
1920    try:
1921        import uv
1922        uv_bin = uv.find_uv_bin()
1923    except (ImportError, FileNotFoundError):
1924        uv_bin = None
1925
1926    if uv_bin is None:
1927        return False
1928
1929    return True
1930
1931
1932def is_uv_enabled() -> bool:
1933    """
1934    Return whether the user has disabled `uv`.
1935    """
1936    from meerschaum.utils.misc import is_android
1937    if is_android():
1938        return False
1939
1940    try:
1941        import yaml
1942    except ImportError:
1943        return False
1944
1945    from meerschaum.config import get_config
1946    enabled = get_config('system', 'experimental', 'uv_pip')
1947    return enabled