meerschaum.utils.packages

   1#! /usr/bin/env python
   2# -*- coding: utf-8 -*-
   3# vim:fenc=utf-8
   4
   5"""
   6Functions for managing packages and virtual environments reside here.
   7"""
   8
   9from __future__ import annotations
  10
  11import importlib.util, os, pathlib, re
  12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
  13from meerschaum.utils.threading import Lock, RLock
  14from meerschaum.utils.packages._packages import packages, all_packages, get_install_names
  15from meerschaum.utils.venv import (
  16    activate_venv,
  17    deactivate_venv,
  18    venv_executable,
  19    venv_exec,
  20    venv_exists,
  21    venv_target_path,
  22    inside_venv,
  23    Venv,
  24    init_venv,
  25)
  26
  27_import_module = importlib.import_module
  28_import_hook_venv = None
  29_locks = {
  30    '_pkg_resources_get_distribution': RLock(),
  31    'import_versions': RLock(),
  32    '_checked_for_updates': RLock(),
  33    '_is_installed_first_check': RLock(),
  34    'emitted_pandas_warning': RLock(),
  35}
  36_checked_for_updates = set()
  37_is_installed_first_check: Dict[str, bool] = {}
  38_MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/"
  39
  40def get_module_path(
  41        import_name: str,
  42        venv: Optional[str] = 'mrsm',
  43        debug: bool = False,
  44        _try_install_name_on_fail: bool = True,
  45    ) -> Union[pathlib.Path, None]:
  46    """
  47    Get a module's path without importing.
  48    """
  49    import site
  50    if debug:
  51        from meerschaum.utils.debug import dprint
  52    if not _try_install_name_on_fail:
  53        install_name = _import_to_install_name(import_name, with_version=False)
  54        install_name_lower = install_name.lower().replace('-', '_')
  55        import_name_lower = install_name_lower
  56    else:
  57        import_name_lower = import_name.lower().replace('-', '_')
  58
  59    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
  60    if not vtp.exists():
  61        if debug:
  62            dprint(
  63                (
  64                    "Venv '{venv}' does not exist, cannot import "
  65                    + f"'{import_name}'."
  66                ),
  67                color = False,
  68            )
  69        return None
  70
  71    venv_target_candidate_paths = [vtp]
  72    if venv is None:
  73        site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())]
  74        site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
  75
  76        paths_to_add = [
  77            path
  78            for path in site_user_packages_dirs + site_packages_dirs
  79            if path not in venv_target_candidate_paths
  80        ]
  81        venv_target_candidate_paths += paths_to_add
  82
  83    candidates = []
  84    for venv_target_candidate in venv_target_candidate_paths:
  85        try:
  86            file_names = os.listdir(venv_target_candidate)
  87        except FileNotFoundError:
  88            continue
  89        for file_name in file_names:
  90            file_name_lower = file_name.lower().replace('-', '_')
  91            if not file_name_lower.startswith(import_name_lower):
  92                continue
  93            if file_name.endswith('dist_info'):
  94                continue
  95            file_path = venv_target_candidate / file_name
  96
  97            ### Most likely: Is a directory with __init__.py
  98            if file_name_lower == import_name_lower and file_path.is_dir():
  99                init_path = file_path / '__init__.py'
 100                if init_path.exists():
 101                    candidates.append(init_path)
 102
 103            ### May be a standalone .py file.
 104            elif file_name_lower == import_name_lower + '.py':
 105                candidates.append(file_path)
 106
 107            ### Compiled wheels (e.g. pyodbc)
 108            elif file_name_lower.startswith(import_name_lower + '.'):
 109                candidates.append(file_path)
 110
 111    if len(candidates) == 1:
 112        return candidates[0]
 113
 114    if not candidates:
 115        if _try_install_name_on_fail:
 116            return get_module_path(
 117                import_name, venv=venv, debug=debug,
 118                _try_install_name_on_fail=False
 119            )
 120        return None
 121
 122    specs_paths = []
 123    for candidate_path in candidates:
 124        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
 125        if spec is not None:
 126            return candidate_path
 127    
 128    return None
 129
 130
 131def manually_import_module(
 132        import_name: str,
 133        venv: Optional[str] = 'mrsm',
 134        check_update: bool = True,
 135        check_pypi: bool = False,
 136        install: bool = True,
 137        split: bool = True,
 138        warn: bool = True,
 139        color: bool = True,
 140        debug: bool = False,
 141        use_sys_modules: bool = True,
 142    ) -> Union['ModuleType', None]:
 143    """
 144    Manually import a module from a virtual environment (or the base environment).
 145
 146    Parameters
 147    ----------
 148    import_name: str
 149        The name of the module.
 150        
 151    venv: Optional[str], default 'mrsm'
 152        The virtual environment to read from.
 153
 154    check_update: bool, default True
 155        If `True`, examine whether the available version of the package meets the required version.
 156
 157    check_pypi: bool, default False
 158        If `True`, check PyPI for updates before importing.
 159
 160    install: bool, default True
 161        If `True`, install the package if it's not installed or needs an update.
 162
 163    split: bool, default True
 164        If `True`, split `import_name` on periods to get the package name.
 165
 166    warn: bool, default True
 167        If `True`, raise a warning if the package cannot be imported.
 168
 169    color: bool, default True
 170        If `True`, use color output for debug and warning text.
 171
 172    debug: bool, default False
 173        Verbosity toggle.
 174
 175    use_sys_modules: bool, default True
 176        If `True`, return the module in `sys.modules` if it exists.
 177        Otherwise continue with manually importing.
 178
 179    Returns
 180    -------
 181    The specified module or `None` if it can't be imported.
 182
 183    """
 184    import sys
 185    _previously_imported = import_name in sys.modules
 186    if _previously_imported and use_sys_modules:
 187        return sys.modules[import_name]
 188    if debug:
 189        from meerschaum.utils.debug import dprint
 190    from meerschaum.utils.warnings import warn as warn_function
 191    import warnings
 192    root_name = import_name.split('.')[0] if split else import_name
 193    install_name = _import_to_install_name(root_name)
 194
 195    root_path = get_module_path(root_name, venv=venv)
 196    if root_path is None:
 197        return None
 198    mod_path = root_path
 199    if mod_path.is_dir():
 200        for _dir in import_name.split('.')[:-1]:
 201            mod_path = mod_path / _dir
 202            possible_end_module_filename = import_name.split('.')[-1] + '.py'
 203            try:
 204                mod_path = (
 205                    (mod_path / possible_end_module_filename)
 206                    if possible_end_module_filename in os.listdir(mod_path)
 207                    else (
 208                        mod_path / import_name.split('.')[-1] / '__init__.py'
 209                    )
 210                )
 211            except Exception as e:
 212                mod_path = None
 213
 214    spec = (
 215        importlib.util.find_spec(import_name) if mod_path is None or not mod_path.exists()
 216        else importlib.util.spec_from_file_location(import_name, str(mod_path))
 217    )
 218    root_spec = (
 219        importlib.util.find_spec(root_name) if not root_path.exists()
 220        else importlib.util.spec_from_file_location(root_name, str(root_path))
 221    )
 222
 223    ### Check for updates before importing.
 224    _version = (
 225        determine_version(
 226            pathlib.Path(root_spec.origin),
 227            import_name=root_name, venv=venv, debug=debug
 228        ) if root_spec is not None and root_spec.origin is not None else None
 229    )
 230
 231    if _version is not None:
 232        if check_update:
 233            if need_update(
 234                None,
 235                import_name = root_name,
 236                version = _version,
 237                check_pypi = check_pypi,
 238                debug = debug,
 239            ):
 240                if install:
 241                    if not pip_install(
 242                        root_name,
 243                        venv = venv,
 244                        split = False,
 245                        check_update = check_update,
 246                        color = color,
 247                        debug = debug
 248                    ) and warn:
 249                        warn_function(
 250                            f"There's an update available for '{install_name}', "
 251                            + "but it failed to install. "
 252                            + "Try installig via Meerschaum with "
 253                            + "`install packages '{install_name}'`.",
 254                            ImportWarning,
 255                            stacklevel = 3,
 256                            color = False,
 257                        )
 258                elif warn:
 259                    warn_function(
 260                        f"There's an update available for '{root_name}'.",
 261                        stack = False,
 262                        color = False,
 263                    )
 264                spec = (
 265                    importlib.util.find_spec(import_name)
 266                    if mod_path is None or not mod_path.exists()
 267                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
 268                )
 269
 270
 271    if spec is None:
 272        try:
 273            mod = _import_module(import_name)
 274        except Exception as e:
 275            mod = None
 276        return mod
 277
 278    with Venv(venv, debug=debug):
 279        mod = importlib.util.module_from_spec(spec)
 280        old_sys_mod = sys.modules.get(import_name, None)
 281        sys.modules[import_name] = mod
 282
 283        try:
 284            with warnings.catch_warnings():
 285                warnings.filterwarnings('ignore', 'The NumPy')
 286                spec.loader.exec_module(mod)
 287        except Exception as e:
 288            pass
 289        mod = _import_module(import_name)
 290        if old_sys_mod is not None:
 291            sys.modules[import_name] = old_sys_mod
 292        else:
 293            del sys.modules[import_name]
 294    
 295    return mod
 296
 297
 298def _import_to_install_name(import_name: str, with_version: bool = True) -> str:
 299    """
 300    Try to translate an import name to an installation name.
 301    """
 302    install_name = all_packages.get(import_name, import_name)
 303    if with_version:
 304        return install_name
 305    return get_install_no_version(install_name)
 306
 307
 308def _import_to_dir_name(import_name: str) -> str:
 309    """
 310    Translate an import name to the package name in the sites-packages directory.
 311    """
 312    import re
 313    return re.split(
 314        r'[<>=\[]', all_packages.get(import_name, import_name)
 315    )[0].replace('-', '_').lower() 
 316
 317
 318def _install_to_import_name(install_name: str) -> str:
 319    """
 320    Translate an installation name to a package's import name.
 321    """
 322    _install_no_version = get_install_no_version(install_name)
 323    return get_install_names().get(_install_no_version, _install_no_version)
 324
 325
 326def get_install_no_version(install_name: str) -> str:
 327    """
 328    Strip the version information from the install name.
 329    """
 330    import re
 331    return re.split(r'[\[=<>,! \]]', install_name)[0]
 332
 333
 334import_versions = {}
 335def determine_version(
 336        path: pathlib.Path,
 337        import_name: Optional[str] = None,
 338        venv: Optional[str] = 'mrsm',
 339        search_for_metadata: bool = True,
 340        split: bool = True,
 341        warn: bool = False,
 342        debug: bool = False,
 343    ) -> Union[str, None]:
 344    """
 345    Determine a module's `__version__` string from its filepath.
 346    
 347    First it searches for pip metadata, then it attempts to import the module in a subprocess.
 348
 349    Parameters
 350    ----------
 351    path: pathlib.Path
 352        The file path of the module.
 353
 354    import_name: Optional[str], default None
 355        The name of the module. If omitted, it will be determined from the file path.
 356        Defaults to `None`.
 357
 358    venv: Optional[str], default 'mrsm'
 359        The virtual environment of the Python interpreter to use if importing is necessary.
 360
 361    search_for_metadata: bool, default True
 362        If `True`, search the pip site_packages directory (assumed to be the parent)
 363        for the corresponding dist-info directory.
 364
 365    warn: bool, default True
 366        If `True`, raise a warning if the module fails to import in the subprocess.
 367
 368    split: bool, default True
 369        If `True`, split the determined import name by periods to get the room name.
 370
 371    Returns
 372    -------
 373    The package's version string if available or `None`.
 374    If multiple versions are found, it will trigger an import in a subprocess.
 375
 376    """
 377    with _locks['import_versions']:
 378        if venv not in import_versions:
 379            import_versions[venv] = {}
 380    import importlib.metadata
 381    import re, os
 382    old_cwd = os.getcwd()
 383    if debug:
 384        from meerschaum.utils.debug import dprint
 385    from meerschaum.utils.warnings import warn as warn_function
 386    if import_name is None:
 387        import_name = path.parent.stem if path.stem == '__init__' else path.stem
 388        import_name = import_name.split('.')[0] if split else import_name
 389    if import_name in import_versions[venv]:
 390        return import_versions[venv][import_name]
 391    _version = None
 392    module_parent_dir = (
 393        path.parent.parent if path.stem == '__init__' else path.parent
 394    ) if path is not None else venv_target_path(venv, debug=debug)
 395
 396    installed_dir_name = _import_to_dir_name(import_name)
 397    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
 398
 399    ### First, check if a dist-info directory exists.
 400    _found_versions = []
 401    if search_for_metadata:
 402        for filename in os.listdir(module_parent_dir):
 403            if not filename.endswith('.dist-info'):
 404                continue
 405            filename_lower = filename.lower()
 406            if not filename_lower.startswith(clean_installed_dir_name + '-'):
 407                continue
 408            _v = filename.replace('.dist-info', '').split("-")[-1]
 409            _found_versions.append(_v)
 410
 411    if len(_found_versions) == 1:
 412        _version = _found_versions[0]
 413        with _locks['import_versions']:
 414            import_versions[venv][import_name] = _version
 415        return _found_versions[0]
 416
 417    if not _found_versions:
 418        try:
 419            import importlib.metadata as importlib_metadata
 420        except ImportError:
 421            importlib_metadata = attempt_import(
 422                'importlib_metadata',
 423                debug=debug, check_update=False, precheck=False,
 424                color=False, check_is_installed=False, lazy=False,
 425            )
 426        try:
 427            os.chdir(module_parent_dir)
 428            _version = importlib_metadata.metadata(import_name)['Version']
 429        except Exception as e:
 430            _version = None
 431        finally:
 432            os.chdir(old_cwd)
 433
 434        if _version is not None:
 435            with _locks['import_versions']:
 436                import_versions[venv][import_name] = _version
 437            return _version
 438
 439    if debug:
 440        print(f'Found multiple versions for {import_name}: {_found_versions}')
 441
 442    module_parent_dir_str = module_parent_dir.as_posix()
 443
 444    ### Not a pip package, so let's try importing the module directly (in a subprocess).
 445    _no_version_str = 'no-version'
 446    code = (
 447        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
 448        + f"module = importlib.import_module('{import_name}');\n"
 449        + "try:\n"
 450        + "  print(module.__version__ , end='')\n"
 451        + "except:\n"
 452        + f"  print('{_no_version_str}', end='')"
 453    )
 454    exit_code, stdout_bytes, stderr_bytes = venv_exec(
 455        code, venv=venv, with_extras=True, debug=debug
 456    )
 457    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
 458    _version = stdout.split('\n')[-1] if exit_code == 0 else None
 459    _version = _version if _version != _no_version_str else None
 460
 461    if _version is None:
 462        _version = _get_package_metadata(import_name, venv).get('version', None)
 463    if _version is None and warn:
 464        warn_function(
 465            f"Failed to determine a version for '{import_name}':\n{stderr}",
 466            stack = False
 467        )
 468
 469    ### If `__version__` doesn't exist, return `None`.
 470    import_versions[venv][import_name] = _version
 471    return _version
 472
 473
 474def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]:
 475    """
 476    Get a package's metadata from pip.
 477    This is useful for getting a version when no `__version__` is defined
 478    and multiple versions are installed.
 479
 480    Parameters
 481    ----------
 482    import_name: str
 483        The package's import or installation name.
 484
 485    venv: Optional[str]
 486        The virtual environment which contains the package.
 487
 488    Returns
 489    -------
 490    A dictionary of metadata from pip.
 491    """
 492    import re
 493    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 494    install_name = _import_to_install_name(import_name)
 495    _args = ['pip', 'show', install_name]
 496    if venv is not None:
 497        cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 498        _args += ['--cache-dir', cache_dir_path.as_posix()]
 499
 500    proc = run_python_package(
 501        'uv', _args,
 502        capture_output=True, as_proc=True, venv=venv, universal_newlines=True,
 503    )
 504    outs, errs = proc.communicate()
 505    lines = outs.split('\n')
 506    meta = {}
 507    for line in lines:
 508        vals = line.split(": ")
 509        if len(vals) != 2:
 510            continue
 511        k, v = vals[0].lower(), vals[1]
 512        if v and 'UNKNOWN' not in v:
 513            meta[k] = v
 514    return meta
 515
 516
 517def need_update(
 518        package: Optional['ModuleType'] = None,
 519        install_name: Optional[str] = None,
 520        import_name: Optional[str] = None,
 521        version: Optional[str] = None,
 522        check_pypi: bool = False,
 523        split: bool = True,
 524        color: bool = True,
 525        debug: bool = False,
 526        _run_determine_version: bool = True,
 527    ) -> bool:
 528    """
 529    Check if a Meerschaum dependency needs an update.
 530    Returns a bool for whether or not a package needs to be updated.
 531
 532    Parameters
 533    ----------
 534    package: 'ModuleType'
 535        The module of the package to be updated.
 536
 537    install_name: Optional[str], default None
 538        If provided, use this string to determine the required version.
 539        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
 540
 541    import_name:
 542        If provided, override the package's `__name__` string.
 543
 544    version: Optional[str], default None
 545        If specified, override the package's `__version__` string.
 546
 547    check_pypi: bool, default False
 548        If `True`, check pypi.org for updates.
 549        Defaults to `False`.
 550
 551    split: bool, default True
 552        If `True`, split the module's name on periods to detrive the root name.
 553        Defaults to `True`.
 554
 555    color: bool, default True
 556        If `True`, format debug output.
 557        Defaults to `True`.
 558
 559    debug: bool, default True
 560        Verbosity toggle.
 561
 562    Returns
 563    -------
 564    A bool indicating whether the package requires an update.
 565
 566    """
 567    if debug:
 568        from meerschaum.utils.debug import dprint
 569    from meerschaum.utils.warnings import warn as warn_function
 570    import re
 571    root_name = (
 572        package.__name__.split('.')[0] if split else package.__name__
 573    ) if import_name is None else (
 574        import_name.split('.')[0] if split else import_name
 575    )
 576    install_name = install_name or _import_to_install_name(root_name)
 577    with _locks['_checked_for_updates']:
 578        if install_name in _checked_for_updates:
 579            return False
 580        _checked_for_updates.add(install_name)
 581
 582    _install_no_version = get_install_no_version(install_name)
 583    required_version = install_name.replace(_install_no_version, '')
 584    if ']' in required_version:
 585        required_version = required_version.split(']')[1]
 586
 587    ### No minimum version was specified, and we're not going to check PyPI.
 588    if not required_version and not check_pypi:
 589        return False
 590
 591    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
 592    if '.dev' in required_version:
 593        required_version = required_version.split('.dev')[0]
 594    if version and '.dev' in version:
 595        version = version.split('.dev')[0]
 596
 597    try:
 598        if not version:
 599            if not _run_determine_version:
 600                version = determine_version(
 601                    pathlib.Path(package.__file__),
 602                    import_name=root_name, warn=False, debug=debug
 603                )
 604        if version is None:
 605            return False
 606    except Exception as e:
 607        if debug:
 608            dprint(str(e), color=color)
 609            dprint("No version could be determined from the installed package.", color=color)
 610        return False
 611    split_version = version.split('.')
 612    last_part = split_version[-1]
 613    if len(split_version) == 2:
 614        version = '.'.join(split_version) + '.0'
 615    elif 'dev' in last_part or 'rc' in last_part:
 616        tag = 'dev' if 'dev' in last_part else 'rc'
 617        last_sep = '-'
 618        if not last_part.startswith(tag):
 619            last_part = f'-{tag}'.join(last_part.split(tag))
 620            last_sep = '.'
 621        version = '.'.join(split_version[:-1]) + last_sep + last_part
 622    elif len(split_version) > 3:
 623        version = '.'.join(split_version[:3])
 624
 625    packaging_version = attempt_import(
 626        'packaging.version', check_update=False, lazy=False, debug=debug,
 627    )
 628
 629    ### Get semver if necessary
 630    if required_version:
 631        semver_path = get_module_path('semver', debug=debug)
 632        if semver_path is None:
 633            pip_install(_import_to_install_name('semver'), debug=debug)
 634        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
 635    if check_pypi:
 636        ### Check PyPI for updates
 637        update_checker = attempt_import(
 638            'update_checker', lazy=False, check_update=False, debug=debug
 639        )
 640        checker = update_checker.UpdateChecker()
 641        result = checker.check(_install_no_version, version)
 642    else:
 643        ### Skip PyPI and assume we can't be sure.
 644        result = None
 645
 646    ### Compare PyPI's version with our own.
 647    if result is not None:
 648        ### We have a result from PyPI and a stated required version.
 649        if required_version:
 650            try:
 651                return semver.Version.parse(result.available_version).match(required_version)
 652            except AttributeError as e:
 653                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 654                semver = manually_import_module('semver', venv='mrsm')
 655                return semver.Version.parse(version).match(required_version)
 656            except Exception as e:
 657                if debug:
 658                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
 659                return False
 660
 661        ### If `check_pypi` and we don't have a required version, check if PyPI's version
 662        ### is newer than the installed version.
 663        else:
 664            return (
 665                packaging_version.parse(result.available_version) > 
 666                packaging_version.parse(version)
 667            )
 668
 669    ### We might be depending on a prerelease.
 670    ### Sanity check that the required version is not greater than the installed version. 
 671    required_version = (
 672        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
 673        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
 674        .replace('-py3-none-any.whl', '')
 675    )
 676
 677    if 'a' in required_version:
 678        required_version = required_version.replace('a', '-dev').replace('+mrsm', '')
 679        version = version.replace('a', '-dev').replace('+mrsm', '')
 680    try:
 681        return (
 682            (not semver.Version.parse(version).match(required_version))
 683            if required_version else False
 684        )
 685    except AttributeError as e:
 686        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 687        semver = manually_import_module('semver', venv='mrsm', debug=debug)
 688        return (
 689            (not semver.Version.parse(version).match(required_version))
 690            if required_version else False
 691        )
 692    except Exception as e:
 693        print(f"Unable to parse version ({version}) for package '{import_name}'.")
 694        print(e)
 695        if debug:
 696            dprint(e)
 697        return False
 698    try:
 699        return (
 700            packaging_version.parse(version) > 
 701            packaging_version.parse(required_version)
 702        )
 703    except Exception as e:
 704        if debug:
 705            dprint(e)
 706        return False
 707    return False
 708
 709
 710def get_pip(
 711        venv: Optional[str] = 'mrsm',
 712        color: bool = True,
 713        debug: bool = False,
 714    ) -> bool:
 715    """
 716    Download and run the get-pip.py script.
 717
 718    Parameters
 719    ----------
 720    venv: Optional[str], default 'mrsm'
 721        The virtual environment into which to install `pip`.
 722
 723    color: bool, default True
 724        If `True`, force color output.
 725
 726    debug: bool, default False
 727        Verbosity toggle.
 728
 729    Returns
 730    -------
 731    A bool indicating success.
 732
 733    """
 734    import sys, subprocess
 735    from meerschaum.utils.misc import wget
 736    from meerschaum.config._paths import CACHE_RESOURCES_PATH
 737    from meerschaum.config.static import STATIC_CONFIG
 738    url = STATIC_CONFIG['system']['urls']['get-pip.py']
 739    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
 740    try:
 741        wget(url, dest, color=False, debug=debug)
 742    except Exception as e:
 743        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
 744        sys.exit(1)
 745    if venv is not None:
 746        init_venv(venv=venv, debug=debug)
 747    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
 748    return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0
 749
 750
 751def pip_install(
 752        *install_names: str,
 753        args: Optional[List[str]] = None,
 754        requirements_file_path: Union[pathlib.Path, str, None] = None,
 755        venv: Optional[str] = 'mrsm',
 756        split: bool = False,
 757        check_update: bool = True,
 758        check_pypi: bool = True,
 759        check_wheel: bool = True,
 760        _uninstall: bool = False,
 761        _from_completely_uninstall: bool = False,
 762        _install_uv_pip: bool = True,
 763        color: bool = True,
 764        silent: bool = False,
 765        debug: bool = False,
 766    ) -> bool:
 767    """
 768    Install packages from PyPI with `pip`.
 769
 770    Parameters
 771    ----------
 772    *install_names: str
 773        The installation names of packages to be installed.
 774        This includes version restrictions.
 775        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 776        from its import name.
 777        
 778    args: Optional[List[str]], default None
 779        A list of command line arguments to pass to `pip`.
 780        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 781
 782    requirements_file_path: Optional[pathlib.Path, str], default None
 783        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 784
 785    venv: str, default 'mrsm'
 786        The virtual environment to install into.
 787
 788    split: bool, default False
 789        If `True`, split on periods and only install the root package name.
 790
 791    check_update: bool, default True
 792        If `True`, check if the package requires an update.
 793
 794    check_pypi: bool, default True
 795        If `True` and `check_update` is `True`, check PyPI for the latest version.
 796
 797    check_wheel: bool, default True
 798        If `True`, check if `wheel` is available.
 799
 800    _uninstall: bool, default False
 801        If `True`, uninstall packages instead.
 802
 803    color: bool, default True
 804        If `True`, include color in debug text.
 805
 806    silent: bool, default False
 807        If `True`, skip printing messages.
 808
 809    debug: bool, default False
 810        Verbosity toggle.
 811
 812    Returns
 813    -------
 814    A bool indicating success.
 815
 816    """
 817    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 818    from meerschaum.config import get_config
 819    from meerschaum.config.static import STATIC_CONFIG
 820    from meerschaum.utils.warnings import warn
 821    from meerschaum.utils.misc import is_android
 822    if args is None:
 823        args = ['--upgrade'] if not _uninstall else []
 824    if color:
 825        ANSI, UNICODE = True, True
 826    else:
 827        ANSI, UNICODE = False, False
 828    if check_wheel:
 829        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 830
 831    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
 832    inside_daemon = daemon_env_var in os.environ
 833    if inside_daemon:
 834        silent = True
 835
 836    _args = list(args)
 837    have_pip = venv_contains_package('pip', venv=None, debug=debug)
 838    try:
 839        import pip
 840        have_pip = True
 841    except ImportError:
 842        have_pip = False
 843    try:
 844        import uv
 845        uv_bin = uv.find_uv_bin()
 846        have_uv_pip = True
 847    except (ImportError, FileNotFoundError):
 848        uv_bin = None
 849        have_uv_pip = False
 850    if have_pip and not have_uv_pip and _install_uv_pip and not is_android():
 851        if not pip_install(
 852            'uv',
 853            venv=None,
 854            debug=debug,
 855            _install_uv_pip=False,
 856            check_update=False,
 857            check_pypi=False,
 858            check_wheel=False,
 859        ) and not silent:
 860            warn(
 861                f"Failed to install `uv` for virtual environment '{venv}'.",
 862                color=False,
 863            )
 864
 865    use_uv_pip = (
 866        venv_contains_package('uv', venv=None, debug=debug)
 867        and uv_bin is not None
 868        and venv is not None
 869    )
 870
 871    import sys
 872    if not have_pip and not use_uv_pip:
 873        if not get_pip(venv=venv, color=color, debug=debug):
 874            import sys
 875            minor = sys.version_info.minor
 876            print(
 877                "\nFailed to import `pip` and `ensurepip`.\n"
 878                + "If you are running Ubuntu/Debian, "
 879                + "you might need to install `python3.{minor}-distutils`:\n\n"
 880                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
 881                + "Please install pip and restart Meerschaum.\n\n"
 882                + "You can find instructions on installing `pip` here:\n"
 883                + "https://pip.pypa.io/en/stable/installing/"
 884            )
 885            sys.exit(1)
 886    
 887    with Venv(venv, debug=debug):
 888        if venv is not None:
 889            if (
 890                '--ignore-installed' not in args
 891                and '-I' not in _args
 892                and not _uninstall
 893                and not use_uv_pip
 894            ):
 895                _args += ['--ignore-installed']
 896            if '--cache-dir' not in args and not _uninstall:
 897                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 898                _args += ['--cache-dir', str(cache_dir_path)]
 899
 900        if 'pip' not in ' '.join(_args) and not use_uv_pip:
 901            if check_update and not _uninstall:
 902                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
 903                if need_update(pip, check_pypi=check_pypi, debug=debug):
 904                    _args.append(all_packages['pip'])
 905        
 906        _args = (['install'] if not _uninstall else ['uninstall']) + _args
 907
 908        if check_wheel and not _uninstall and not use_uv_pip:
 909            if not have_wheel:
 910                if not pip_install(
 911                    'setuptools', 'wheel', 'uv',
 912                    venv = venv,
 913                    check_update = False,
 914                    check_pypi = False,
 915                    check_wheel = False,
 916                    debug = debug,
 917                    _install_uv_pip = False,
 918                ) and not silent:
 919                    warn(
 920                        (
 921                            "Failed to install `setuptools`, `wheel`, and `uv` for virtual "
 922                            + f"environment '{venv}'."
 923                        ),
 924                        color=False,
 925                    )
 926
 927        if requirements_file_path is not None:
 928            _args.append('-r')
 929            _args.append(pathlib.Path(requirements_file_path).resolve().as_posix())
 930
 931        if not ANSI and '--no-color' not in _args:
 932            _args.append('--no-color')
 933
 934        if '--no-input' not in _args and not use_uv_pip:
 935            _args.append('--no-input')
 936
 937        if _uninstall and '-y' not in _args and not use_uv_pip:
 938            _args.append('-y')
 939
 940        if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip:
 941            _args.append('--no-warn-conflicts')
 942
 943        if '--disable-pip-version-check' not in _args and not use_uv_pip:
 944            _args.append('--disable-pip-version-check')
 945
 946        if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
 947            if venv is not None:
 948                _args += ['--target', venv_target_path(venv, debug=debug)]
 949        elif (
 950            '--target' not in _args
 951                and '-t' not in _args
 952                and not inside_venv()
 953                and not _uninstall
 954                and not use_uv_pip
 955        ):
 956            _args += ['--user']
 957
 958        if debug:
 959            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
 960                if use_uv_pip:
 961                    _args.append('--verbose')
 962        else:
 963            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
 964                pass
 965
 966        _packages = [
 967            (install_name if not _uninstall else get_install_no_version(install_name))
 968            for install_name in install_names
 969        ]
 970        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
 971        for p in _packages:
 972            msg += f'\n  - {p}'
 973        if not silent:
 974            print(msg)
 975
 976        if _uninstall and not _from_completely_uninstall and not use_uv_pip:
 977            for install_name in _packages:
 978                _install_no_version = get_install_no_version(install_name)
 979                if _install_no_version in ('pip', 'wheel', 'uv'):
 980                    continue
 981                if not completely_uninstall_package(
 982                    _install_no_version,
 983                    venv=venv, debug=debug,
 984                ) and not silent:
 985                    warn(
 986                        f"Failed to clean up package '{_install_no_version}'.",
 987                    )
 988
 989        ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease.
 990        if use_uv_pip:
 991            _args.insert(0, 'pip')
 992            if not _uninstall and get_prerelease_dependencies(_packages):
 993                _args.append('--prerelease=allow')
 994
 995        rc = run_python_package(
 996            ('pip' if not use_uv_pip else 'uv'),
 997            _args + _packages,
 998            venv=None,
 999            env=_get_pip_os_env(color=color),
1000            debug=debug,
1001        )
1002        if debug:
1003            print(f"{rc=}")
1004        success = rc == 0
1005
1006    msg = (
1007        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
1008        else "Failed to " + ('un' if _uninstall else '') + "install packages."
1009    )
1010    if not silent:
1011        print(msg)
1012    if debug and not silent:
1013        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
1014    return success
1015
1016
1017def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1018    """
1019    Return a list of explicitly prerelease dependencies from a list of packages.
1020    """
1021    if _packages is None:
1022        _packages = list(all_packages.keys())
1023    prelrease_strings = ['dev', 'rc', 'a']
1024    prerelease_packages = []
1025    for install_name in _packages:
1026        _install_no_version = get_install_no_version(install_name)
1027        import_name = _install_to_import_name(install_name)
1028        install_with_version = _import_to_install_name(import_name)
1029        version_only = (
1030            install_with_version.lower().replace(_install_no_version.lower(), '')
1031            .split(']')[-1]
1032        )
1033
1034        is_prerelease = False
1035        for prelrease_string in prelrease_strings:
1036            if prelrease_string in version_only:
1037                is_prerelease = True
1038
1039        if is_prerelease:
1040            prerelease_packages.append(install_name)
1041    return prerelease_packages
1042
1043
1044def completely_uninstall_package(
1045        install_name: str,
1046        venv: str = 'mrsm',
1047        debug: bool = False,
1048    ) -> bool:
1049    """
1050    Continue calling `pip uninstall` until a package is completely
1051    removed from a virtual environment. 
1052    This is useful for dealing with multiple installed versions of a package.
1053    """
1054    attempts = 0
1055    _install_no_version = get_install_no_version(install_name)
1056    clean_install_no_version = _install_no_version.lower().replace('-', '_')
1057    installed_versions = []
1058    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1059    if not vtp.exists():
1060        return True
1061
1062    for file_name in os.listdir(vtp):
1063        if not file_name.endswith('.dist-info'):
1064            continue
1065        clean_dist_info = file_name.replace('-', '_').lower()
1066        if not clean_dist_info.startswith(clean_install_no_version):
1067            continue
1068        installed_versions.append(file_name)
1069
1070    max_attempts = len(installed_versions)
1071    while attempts < max_attempts:
1072        if not venv_contains_package(
1073            _install_to_import_name(_install_no_version),
1074            venv=venv, debug=debug,
1075        ):
1076            return True
1077        if not pip_uninstall(
1078            _install_no_version,
1079            venv = venv,
1080            silent = (not debug),
1081            _from_completely_uninstall = True,
1082            debug = debug,
1083        ):
1084            return False
1085        attempts += 1
1086    return False
1087
1088
1089def pip_uninstall(
1090        *args, **kw
1091    ) -> bool:
1092    """
1093    Uninstall Python packages.
1094    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
1095    """
1096    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
1097
1098
1099def run_python_package(
1100        package_name: str,
1101        args: Optional[List[str]] = None,
1102        venv: Optional[str] = 'mrsm',
1103        cwd: Optional[str] = None,
1104        foreground: bool = False,
1105        as_proc: bool = False,
1106        capture_output: bool = False,
1107        debug: bool = False,
1108        **kw: Any,
1109    ) -> Union[int, subprocess.Popen, None]:
1110    """
1111    Runs an installed python package.
1112    E.g. Translates to `/usr/bin/python -m [package]`
1113
1114    Parameters
1115    ----------
1116    package_name: str
1117        The Python module to be executed.
1118
1119    args: Optional[List[str]], default None
1120        Additional command line arguments to be appended after `-m [package]`.
1121
1122    venv: Optional[str], default 'mrsm'
1123        If specified, execute the Python interpreter from a virtual environment.
1124
1125    cwd: Optional[str], default None
1126        If specified, change directories before starting the process.
1127        Defaults to `None`.
1128
1129    as_proc: bool, default False
1130        If `True`, return a `subprocess.Popen` object.
1131
1132    capture_output: bool, default False
1133        If `as_proc` is `True`, capture stdout and stderr.
1134
1135    foreground: bool, default False
1136        If `True`, start the subprocess as a foreground process.
1137        Defaults to `False`.
1138
1139    kw: Any
1140        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1141        and by extension `subprocess.Popen()`.
1142
1143    Returns
1144    -------
1145    Either a return code integer or a `subprocess.Popen` object
1146    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1147    """
1148    import sys, platform
1149    import subprocess
1150    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1151    from meerschaum.utils.process import run_process
1152    from meerschaum.utils.warnings import warn
1153    if args is None:
1154        args = []
1155    old_cwd = os.getcwd()
1156    if cwd is not None:
1157        os.chdir(cwd)
1158    executable = venv_executable(venv=venv)
1159    venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
1160    env_dict = kw.get('env', os.environ).copy()
1161    if venv_path is not None:
1162        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
1163    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1164    import traceback
1165    if debug:
1166        print(command, file=sys.stderr)
1167    try:
1168        to_return = run_process(
1169            command,
1170            foreground = foreground,
1171            as_proc = as_proc,
1172            capture_output = capture_output,
1173            **kw
1174        )
1175    except Exception as e:
1176        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1177        warn(msg, color=False)
1178        stdout, stderr = (
1179            (None, None)
1180            if not capture_output
1181            else (subprocess.PIPE, subprocess.PIPE)
1182        )
1183        proc = subprocess.Popen(
1184            command,
1185            stdout = stdout,
1186            stderr = stderr,
1187            env = env_dict,
1188        )
1189        to_return = proc if as_proc else proc.wait()
1190    except KeyboardInterrupt:
1191        to_return = 1 if not as_proc else None
1192    os.chdir(old_cwd)
1193    return to_return
1194
1195
1196def attempt_import(
1197        *names: str,
1198        lazy: bool = True,
1199        warn: bool = True,
1200        install: bool = True,
1201        venv: Optional[str] = 'mrsm',
1202        precheck: bool = True,
1203        split: bool = True,
1204        check_update: bool = False,
1205        check_pypi: bool = False,
1206        check_is_installed: bool = True,
1207        allow_outside_venv: bool = True,
1208        color: bool = True,
1209        debug: bool = False
1210    ) -> Any:
1211    """
1212    Raise a warning if packages are not installed; otherwise import and return modules.
1213    If `lazy` is `True`, return lazy-imported modules.
1214    
1215    Returns tuple of modules if multiple names are provided, else returns one module.
1216    
1217    Parameters
1218    ----------
1219    names: List[str]
1220        The packages to be imported.
1221
1222    lazy: bool, default True
1223        If `True`, lazily load packages.
1224
1225    warn: bool, default True
1226        If `True`, raise a warning if a package cannot be imported.
1227
1228    install: bool, default True
1229        If `True`, attempt to install a missing package into the designated virtual environment.
1230        If `check_update` is True, install updates if available.
1231
1232    venv: Optional[str], default 'mrsm'
1233        The virtual environment in which to search for packages and to install packages into.
1234
1235    precheck: bool, default True
1236        If `True`, attempt to find module before importing (necessary for checking if modules exist
1237        and retaining lazy imports), otherwise assume lazy is `False`.
1238
1239    split: bool, default True
1240        If `True`, split packages' names on `'.'`.
1241
1242    check_update: bool, default False
1243        If `True` and `install` is `True`, install updates if the required minimum version
1244        does not match.
1245
1246    check_pypi: bool, default False
1247        If `True` and `check_update` is `True`, check PyPI when determining whether
1248        an update is required.
1249
1250    check_is_installed: bool, default True
1251        If `True`, check if the package is contained in the virtual environment.
1252
1253    allow_outside_venv: bool, default True
1254        If `True`, search outside of the specified virtual environment
1255        if the package cannot be found.
1256        Setting to `False` will reinstall the package into a virtual environment, even if it
1257        is installed outside.
1258
1259    color: bool, default True
1260        If `False`, do not print ANSI colors.
1261
1262    Returns
1263    -------
1264    The specified modules. If they're not available and `install` is `True`, it will first
1265    download them into a virtual environment and return the modules.
1266
1267    Examples
1268    --------
1269    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1270    >>> pandas = attempt_import('pandas')
1271
1272    """
1273
1274    import importlib.util
1275
1276    ### to prevent recursion, check if parent Meerschaum package is being imported
1277    if names == ('meerschaum',):
1278        return _import_module('meerschaum')
1279
1280    if venv == 'mrsm' and _import_hook_venv is not None:
1281        if debug:
1282            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1283        venv = _import_hook_venv
1284
1285    _warnings = _import_module('meerschaum.utils.warnings')
1286    warn_function = _warnings.warn
1287
1288    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1289        with Venv(venv=venv, debug=debug):
1290            ### determine the import method (lazy vs normal)
1291            from meerschaum.utils.misc import filter_keywords
1292            import_method = (
1293                _import_module if not lazy
1294                else lazy_import
1295            )
1296            try:
1297                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1298            except Exception as e:
1299                if warn:
1300                    import traceback
1301                    traceback.print_exception(type(e), e, e.__traceback__)
1302                    warn_function(
1303                        f"Failed to import module '{_name}'.\nException:\n{e}",
1304                        ImportWarning,
1305                        stacklevel = (5 if lazy else 4),
1306                        color = False,
1307                    )
1308                mod = None
1309        return mod
1310
1311    modules = []
1312    for name in names:
1313        ### Check if package is a declared dependency.
1314        root_name = name.split('.')[0] if split else name
1315        install_name = _import_to_install_name(root_name)
1316
1317        if install_name is None:
1318            install_name = root_name
1319            if warn and root_name != 'plugins':
1320                warn_function(
1321                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1322                    ImportWarning,
1323                    stacklevel = 3,
1324                    color = False
1325                )
1326
1327        ### Determine if the package exists.
1328        if precheck is False:
1329            found_module = (
1330                do_import(
1331                    name, debug=debug, warn=False, venv=venv, color=color,
1332                    check_update=False, check_pypi=False, split=split,
1333                ) is not None
1334            )
1335        else:
1336            if check_is_installed:
1337                with _locks['_is_installed_first_check']:
1338                    if not _is_installed_first_check.get(name, False):
1339                        package_is_installed = is_installed(
1340                            name,
1341                            venv = venv,
1342                            split = split,
1343                            allow_outside_venv = allow_outside_venv,
1344                            debug = debug,
1345                        )
1346                        _is_installed_first_check[name] = package_is_installed
1347                    else:
1348                        package_is_installed = _is_installed_first_check[name]
1349            else:
1350                package_is_installed = _is_installed_first_check.get(
1351                    name,
1352                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1353                )
1354            found_module = package_is_installed
1355
1356        if not found_module:
1357            if install:
1358                if not pip_install(
1359                    install_name,
1360                    venv = venv,
1361                    split = False,
1362                    check_update = check_update,
1363                    color = color,
1364                    debug = debug
1365                ) and warn:
1366                    warn_function(
1367                        f"Failed to install '{install_name}'.",
1368                        ImportWarning,
1369                        stacklevel = 3,
1370                        color = False,
1371                    )
1372            elif warn:
1373                ### Raise a warning if we can't find the package and install = False.
1374                warn_function(
1375                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1376                     + "some features will not work correctly."
1377                     + f"\n\nSet install=True when calling attempt_import.\n"),
1378                    ImportWarning,
1379                    stacklevel = 3,
1380                    color = False,
1381                )
1382
1383        ### Do the import. Will be lazy if lazy=True.
1384        m = do_import(
1385            name, debug=debug, warn=warn, venv=venv, color=color,
1386            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1387        )
1388        modules.append(m)
1389
1390    modules = tuple(modules)
1391    if len(modules) == 1:
1392        return modules[0]
1393    return modules
1394
1395
1396def lazy_import(
1397        name: str,
1398        local_name: str = None,
1399        **kw
1400    ) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1401    """
1402    Lazily import a package.
1403    """
1404    from meerschaum.utils.packages.lazy_loader import LazyLoader
1405    if local_name is None:
1406        local_name = name
1407    return LazyLoader(
1408        local_name,
1409        globals(),
1410        name,
1411        **kw
1412    )
1413
1414
1415def pandas_name() -> str:
1416    """
1417    Return the configured name for `pandas`.
1418    
1419    Below are the expected possible values:
1420
1421    - 'pandas'
1422    - 'modin.pandas'
1423    - 'dask.dataframe'
1424
1425    """
1426    from meerschaum.config import get_config
1427    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1428    if pandas_module_name == 'modin':
1429        pandas_module_name = 'modin.pandas'
1430    elif pandas_module_name == 'dask':
1431        pandas_module_name = 'dask.dataframe'
1432
1433    return pandas_module_name
1434
1435
1436emitted_pandas_warning: bool = False
1437def import_pandas(
1438        debug: bool = False,
1439        lazy: bool = False,
1440        **kw
1441    ) -> 'ModuleType':
1442    """
1443    Quality-of-life function to attempt to import the configured version of `pandas`.
1444    """
1445    import sys
1446    pandas_module_name = pandas_name()
1447    global emitted_pandas_warning
1448
1449    if pandas_module_name != 'pandas':
1450        with _locks['emitted_pandas_warning']:
1451            if not emitted_pandas_warning:
1452                from meerschaum.utils.warnings import warn
1453                emitted_pandas_warning = True
1454                warn(
1455                    (
1456                        "You are using an alternative Pandas implementation "
1457                        + f"'{pandas_module_name}'"
1458                        + "\n   Features may not work as expected."
1459                    ),
1460                    stack = False,
1461                )
1462
1463    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1464    pandas = attempt_import('pandas', debug=debug, lazy=False, **kw)
1465    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1466    return pd
1467
1468
1469def import_rich(
1470        lazy: bool = True,
1471        debug: bool = False,
1472        **kw : Any
1473    ) -> 'ModuleType':
1474    """
1475    Quality of life function for importing `rich`.
1476    """
1477    from meerschaum.utils.formatting import ANSI, UNICODE
1478    if not ANSI and not UNICODE:
1479        return None
1480
1481    ## need typing_extensions for `from rich import box`
1482    typing_extensions = attempt_import(
1483        'typing_extensions', lazy=False, debug=debug
1484    )
1485    pygments = attempt_import(
1486        'pygments', lazy=False,
1487    )
1488    rich = attempt_import(
1489        'rich', lazy=lazy,
1490        **kw
1491    )
1492    return rich
1493
1494
1495def _dash_less_than_2(**kw) -> bool:
1496    dash = attempt_import('dash', **kw)
1497    if dash is None:
1498        return None
1499    packaging_version = attempt_import('packaging.version', **kw)
1500    return (
1501        packaging_version.parse(dash.__version__) < 
1502        packaging_version.parse('2.0.0')
1503    )
1504
1505
1506def import_dcc(warn=False, **kw) -> 'ModuleType':
1507    """
1508    Import Dash Core Components (`dcc`).
1509    """
1510    return (
1511        attempt_import('dash_core_components', warn=warn, **kw)
1512        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1513    )
1514
1515
1516def import_html(warn=False, **kw) -> 'ModuleType':
1517    """
1518    Import Dash HTML Components (`html`).
1519    """
1520    return (
1521        attempt_import('dash_html_components', warn=warn, **kw)
1522        if _dash_less_than_2(warn=warn, **kw)
1523        else attempt_import('dash.html', warn=warn, **kw)
1524    )
1525
1526
1527def get_modules_from_package(
1528        package: 'package',
1529        names: bool = False,
1530        recursive: bool = False,
1531        lazy: bool = False,
1532        modules_venvs: bool = False,
1533        debug: bool = False
1534    ):
1535    """
1536    Find and import all modules in a package.
1537    
1538    Returns
1539    -------
1540    Either list of modules or tuple of lists.
1541    """
1542    from os.path import dirname, join, isfile, isdir, basename
1543    import glob
1544
1545    pattern = '*' if recursive else '*.py'
1546    package_path = dirname(package.__file__ or package.__path__[0])
1547    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1548    _all = [
1549        basename(f)[:-3] if isfile(f) else basename(f)
1550        for f in module_names
1551            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1552               and not f.endswith('__init__.py')
1553               and not f.endswith('__pycache__')
1554    ]
1555
1556    if debug:
1557        from meerschaum.utils.debug import dprint
1558        dprint(str(_all))
1559    modules = []
1560    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1561        ### there's probably a better way than a try: catch but it'll do for now
1562        try:
1563            ### if specified, activate the module's virtual environment before importing.
1564            ### NOTE: this only considers the filename, so two modules from different packages
1565            ### may end up sharing virtual environments.
1566            if modules_venvs:
1567                activate_venv(module_name.split('.')[-1], debug=debug)
1568            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1569            modules.append(m)
1570        except Exception as e:
1571            if debug:
1572                dprint(str(e))
1573        finally:
1574            if modules_venvs:
1575                deactivate_venv(module_name.split('.')[-1], debug=debug)
1576    if names:
1577        return _all, modules
1578
1579    return modules
1580
1581
1582def import_children(
1583        package: Optional['ModuleType'] = None,
1584        package_name: Optional[str] = None,
1585        types : Optional[List[str]] = None,
1586        lazy: bool = True,
1587        recursive: bool = False,
1588        debug: bool = False
1589    ) -> List['ModuleType']:
1590    """
1591    Import all functions in a package to its `__init__`.
1592
1593    Parameters
1594    ----------
1595    package: Optional[ModuleType], default None
1596        Package to import its functions into.
1597        If `None` (default), use parent.
1598
1599    package_name: Optional[str], default None
1600        Name of package to import its functions into
1601        If None (default), use parent.
1602
1603    types: Optional[List[str]], default None
1604        Types of members to return.
1605        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1606
1607    Returns
1608    -------
1609    A list of modules.
1610    """
1611    import sys, inspect
1612
1613    if types is None:
1614        types = ['method', 'builtin', 'function', 'class', 'module']
1615
1616    ### if package_name and package are None, use parent
1617    if package is None and package_name is None:
1618        package_name = inspect.stack()[1][0].f_globals['__name__']
1619
1620    ### populate package or package_name from other other
1621    if package is None:
1622        package = sys.modules[package_name]
1623    elif package_name is None:
1624        package_name = package.__name__
1625
1626    ### Set attributes in sys module version of package.
1627    ### Kinda like setting a dictionary
1628    ###   functions[name] = func
1629    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1630    _all, members = [], []
1631    objects = []
1632    for module in modules:
1633        _objects = []
1634        for ob in inspect.getmembers(module):
1635            for t in types:
1636                ### ob is a tuple of (name, object)
1637                if getattr(inspect, 'is' + t)(ob[1]):
1638                    _objects.append(ob)
1639
1640        if 'module' in types:
1641            _objects.append((module.__name__.split('.')[0], module))
1642        objects += _objects
1643    for ob in objects:
1644        setattr(sys.modules[package_name], ob[0], ob[1])
1645        _all.append(ob[0])
1646        members.append(ob[1])
1647
1648    if debug:
1649        from meerschaum.utils.debug import dprint
1650        dprint(str(_all))
1651    ### set __all__ for import *
1652    setattr(sys.modules[package_name], '__all__', _all)
1653    return members
1654
1655
1656_reload_module_cache = {}
1657def reload_package(
1658        package: str,
1659        skip_submodules: Optional[List[str]] = None,
1660        lazy: bool = False,
1661        debug: bool = False,
1662        **kw: Any
1663    ):
1664    """
1665    Recursively load a package's subpackages, even if they were not previously loaded.
1666    """
1667    import sys
1668    if isinstance(package, str):
1669        package_name = package
1670    else:
1671        try:
1672            package_name = package.__name__
1673        except Exception as e:
1674            package_name = str(package)
1675
1676    skip_submodules = skip_submodules or []
1677    if 'meerschaum.utils.packages' not in skip_submodules:
1678        skip_submodules.append('meerschaum.utils.packages')
1679    def safeimport():
1680        subs = [
1681            m for m in sys.modules
1682            if m.startswith(package_name + '.')
1683        ]
1684        subs_to_skip = []
1685        for skip_mod in skip_submodules:
1686            for mod in subs:
1687                if mod.startswith(skip_mod):
1688                    subs_to_skip.append(mod)
1689                    continue
1690
1691        subs = [m for m in subs if m not in subs_to_skip]
1692        for module_name in subs:
1693            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1694        if not subs_to_skip:
1695            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1696
1697        return _import_module(package_name)
1698
1699    return safeimport()
1700
1701
1702def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1703    """
1704    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1705    """
1706    reload_package(
1707        'meerschaum',
1708        skip_submodules = [
1709            'meerschaum._internal.shell',
1710            'meerschaum.utils.pool',
1711        ]
1712    )
1713
1714    from meerschaum.plugins import reload_plugins
1715    from meerschaum._internal.shell.Shell import _insert_shell_actions
1716    reload_plugins(debug=debug)
1717    _insert_shell_actions()
1718    return True, "Success"
1719
1720
1721def is_installed(
1722        import_name: str,
1723        venv: Optional[str] = 'mrsm',
1724        split: bool = True,
1725        allow_outside_venv: bool = True,
1726        debug: bool = False,
1727    ) -> bool:
1728    """
1729    Check whether a package is installed.
1730
1731    Parameters
1732    ----------
1733    import_name: str
1734        The import name of the module.
1735
1736    venv: Optional[str], default 'mrsm'
1737        The venv in which to search for the module.
1738
1739    split: bool, default True
1740        If `True`, split on periods to determine the root module name.
1741
1742    allow_outside_venv: bool, default True
1743        If `True`, search outside of the specified virtual environment
1744        if the package cannot be found.
1745    """
1746    if debug:
1747        from meerschaum.utils.debug import dprint
1748    root_name = import_name.split('.')[0] if split else import_name
1749    import importlib.util
1750    with Venv(venv, debug=debug):
1751        try:
1752            spec_path = pathlib.Path(
1753                get_module_path(root_name, venv=venv, debug=debug)
1754                or
1755                (
1756                    importlib.util.find_spec(root_name).origin 
1757                    if venv is not None and allow_outside_venv
1758                    else None
1759                )
1760            )
1761        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1762            spec_path = None
1763
1764        found = (
1765            not need_update(
1766                None, import_name = root_name,
1767                _run_determine_version = False,
1768                check_pypi = False,
1769                version = determine_version(
1770                    spec_path, venv=venv, debug=debug, import_name=root_name
1771                ),
1772                debug = debug,
1773            )
1774        ) if spec_path is not None else False
1775
1776    return found
1777
1778
1779def venv_contains_package(
1780        import_name: str,
1781        venv: Optional[str] = 'mrsm',
1782        split: bool = True,
1783        debug: bool = False,
1784    ) -> bool:
1785    """
1786    Search the contents of a virtual environment for a package.
1787    """
1788    import site
1789    import pathlib
1790    root_name = import_name.split('.')[0] if split else import_name
1791    return get_module_path(root_name, venv=venv, debug=debug) is not None
1792
1793
1794def package_venv(package: 'ModuleType') -> Union[str, None]:
1795    """
1796    Inspect a package and return the virtual environment in which it presides.
1797    """
1798    import os
1799    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1800    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1801        return None
1802    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
1803
1804
1805def ensure_readline() -> 'ModuleType':
1806    """Make sure that the `readline` package is able to be imported."""
1807    import sys
1808    try:
1809        import readline
1810    except ImportError:
1811        readline = None
1812
1813    if readline is None:
1814        import platform
1815        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1816        try:
1817            rl = attempt_import(
1818                rl_name,
1819                lazy = False,
1820                install = True,
1821                venv = None,
1822                warn = False,
1823            )
1824        except (ImportError, ModuleNotFoundError):
1825            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1826                print(f"Unable to import {rl_name}!", file=sys.stderr)
1827                sys.exit(1)
1828
1829    sys.modules['readline'] = readline
1830    return readline
1831
1832_pkg_resources_get_distribution = None
1833_custom_distributions = {}
1834def _monkey_patch_get_distribution(_dist: str, _version: str) -> None:
1835    """
1836    Monkey patch `pkg_resources.get_distribution` to allow for importing `flask_compress`.
1837    """
1838    import pkg_resources
1839    from collections import namedtuple
1840    global _pkg_resources_get_distribution
1841    with _locks['_pkg_resources_get_distribution']:
1842        _pkg_resources_get_distribution = pkg_resources.get_distribution
1843    _custom_distributions[_dist] = _version
1844    _Dist = namedtuple('_Dist', ['version'])
1845    def _get_distribution(dist):
1846        """Hack for flask-compress."""
1847        if dist in _custom_distributions:
1848            return _Dist(_custom_distributions[dist])
1849        return _pkg_resources_get_distribution(dist)
1850    pkg_resources.get_distribution = _get_distribution
1851
1852
1853def _get_pip_os_env(color: bool = True):
1854    """
1855    Return the environment variables context in which `pip` should be run.
1856    See PEP 668 for why we are overriding the environment.
1857    """
1858    import os, sys, platform
1859    python_bin_path = pathlib.Path(sys.executable)
1860    pip_os_env = os.environ.copy()
1861    path_str = pip_os_env.get('PATH', '') or ''
1862    path_sep = ':' if platform.system() != 'Windows' else ';'
1863    pip_os_env.update({
1864        'PIP_BREAK_SYSTEM_PACKAGES': 'true',
1865        ('FORCE_COLOR' if color else 'NO_COLOR'): '1',
1866    })
1867    if str(python_bin_path) not in path_str:
1868        pip_os_env['PATH'] = str(python_bin_path.parent) + path_sep + path_str
1869
1870    return pip_os_env