meerschaum.utils.packages

   1#! /usr/bin/env python
   2# -*- coding: utf-8 -*-
   3# vim:fenc=utf-8
   4
   5"""
   6Functions for managing packages and virtual environments reside here.
   7"""
   8
   9from __future__ import annotations
  10
  11import importlib.util, os, pathlib, re
  12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
  13from meerschaum.utils.threading import Lock, RLock
  14from meerschaum.utils.packages._packages import packages, all_packages, get_install_names
  15from meerschaum.utils.venv import (
  16    activate_venv,
  17    deactivate_venv,
  18    venv_executable,
  19    venv_exec,
  20    venv_exists,
  21    venv_target_path,
  22    inside_venv,
  23    Venv,
  24    init_venv,
  25)
  26
  27_import_module = importlib.import_module
  28_import_hook_venv = None
  29_locks = {
  30    '_pkg_resources_get_distribution': RLock(),
  31    'import_versions': RLock(),
  32    '_checked_for_updates': RLock(),
  33    '_is_installed_first_check': RLock(),
  34    'emitted_pandas_warning': RLock(),
  35}
  36_checked_for_updates = set()
  37_is_installed_first_check: Dict[str, bool] = {}
  38_MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/"
  39
  40def get_module_path(
  41        import_name: str,
  42        venv: Optional[str] = 'mrsm',
  43        debug: bool = False,
  44        _try_install_name_on_fail: bool = True,
  45    ) -> Union[pathlib.Path, None]:
  46    """
  47    Get a module's path without importing.
  48    """
  49    import site
  50    if debug:
  51        from meerschaum.utils.debug import dprint
  52    if not _try_install_name_on_fail:
  53        install_name = _import_to_install_name(import_name, with_version=False)
  54        install_name_lower = install_name.lower().replace('-', '_')
  55        import_name_lower = install_name_lower
  56    else:
  57        import_name_lower = import_name.lower().replace('-', '_')
  58
  59    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
  60    if not vtp.exists():
  61        if debug:
  62            dprint(
  63                (
  64                    "Venv '{venv}' does not exist, cannot import "
  65                    + f"'{import_name}'."
  66                ),
  67                color = False,
  68            )
  69        return None
  70
  71    venv_target_candidate_paths = [vtp]
  72    if venv is None:
  73        site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())]
  74        site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
  75
  76        paths_to_add = [
  77            path
  78            for path in site_user_packages_dirs + site_packages_dirs
  79            if path not in venv_target_candidate_paths
  80        ]
  81        venv_target_candidate_paths += paths_to_add
  82
  83    candidates = []
  84    for venv_target_candidate in venv_target_candidate_paths:
  85        try:
  86            file_names = os.listdir(venv_target_candidate)
  87        except FileNotFoundError:
  88            continue
  89        for file_name in file_names:
  90            file_name_lower = file_name.lower().replace('-', '_')
  91            if not file_name_lower.startswith(import_name_lower):
  92                continue
  93            if file_name.endswith('dist_info'):
  94                continue
  95            file_path = venv_target_candidate / file_name
  96
  97            ### Most likely: Is a directory with __init__.py
  98            if file_name_lower == import_name_lower and file_path.is_dir():
  99                init_path = file_path / '__init__.py'
 100                if init_path.exists():
 101                    candidates.append(init_path)
 102
 103            ### May be a standalone .py file.
 104            elif file_name_lower == import_name_lower + '.py':
 105                candidates.append(file_path)
 106
 107            ### Compiled wheels (e.g. pyodbc)
 108            elif file_name_lower.startswith(import_name_lower + '.'):
 109                candidates.append(file_path)
 110
 111    if len(candidates) == 1:
 112        return candidates[0]
 113
 114    if not candidates:
 115        if _try_install_name_on_fail:
 116            return get_module_path(
 117                import_name, venv=venv, debug=debug,
 118                _try_install_name_on_fail=False
 119            )
 120        return None
 121
 122    specs_paths = []
 123    for candidate_path in candidates:
 124        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
 125        if spec is not None:
 126            return candidate_path
 127    
 128    return None
 129
 130
 131def manually_import_module(
 132        import_name: str,
 133        venv: Optional[str] = 'mrsm',
 134        check_update: bool = True,
 135        check_pypi: bool = False,
 136        install: bool = True,
 137        split: bool = True,
 138        warn: bool = True,
 139        color: bool = True,
 140        debug: bool = False,
 141        use_sys_modules: bool = True,
 142    ) -> Union['ModuleType', None]:
 143    """
 144    Manually import a module from a virtual environment (or the base environment).
 145
 146    Parameters
 147    ----------
 148    import_name: str
 149        The name of the module.
 150        
 151    venv: Optional[str], default 'mrsm'
 152        The virtual environment to read from.
 153
 154    check_update: bool, default True
 155        If `True`, examine whether the available version of the package meets the required version.
 156
 157    check_pypi: bool, default False
 158        If `True`, check PyPI for updates before importing.
 159
 160    install: bool, default True
 161        If `True`, install the package if it's not installed or needs an update.
 162
 163    split: bool, default True
 164        If `True`, split `import_name` on periods to get the package name.
 165
 166    warn: bool, default True
 167        If `True`, raise a warning if the package cannot be imported.
 168
 169    color: bool, default True
 170        If `True`, use color output for debug and warning text.
 171
 172    debug: bool, default False
 173        Verbosity toggle.
 174
 175    use_sys_modules: bool, default True
 176        If `True`, return the module in `sys.modules` if it exists.
 177        Otherwise continue with manually importing.
 178
 179    Returns
 180    -------
 181    The specified module or `None` if it can't be imported.
 182
 183    """
 184    import sys
 185    _previously_imported = import_name in sys.modules
 186    if _previously_imported and use_sys_modules:
 187        return sys.modules[import_name]
 188    if debug:
 189        from meerschaum.utils.debug import dprint
 190    from meerschaum.utils.warnings import warn as warn_function
 191    import warnings
 192    root_name = import_name.split('.')[0] if split else import_name
 193    install_name = _import_to_install_name(root_name)
 194
 195    root_path = get_module_path(root_name, venv=venv)
 196    if root_path is None:
 197        return None
 198    mod_path = root_path
 199    if mod_path.is_dir():
 200        for _dir in import_name.split('.')[:-1]:
 201            mod_path = mod_path / _dir
 202            possible_end_module_filename = import_name.split('.')[-1] + '.py'
 203            try:
 204                mod_path = (
 205                    (mod_path / possible_end_module_filename)
 206                    if possible_end_module_filename in os.listdir(mod_path)
 207                    else (
 208                        mod_path / import_name.split('.')[-1] / '__init__.py'
 209                    )
 210                )
 211            except Exception as e:
 212                mod_path = None
 213
 214    spec = (
 215        importlib.util.find_spec(import_name) if mod_path is None or not mod_path.exists()
 216        else importlib.util.spec_from_file_location(import_name, str(mod_path))
 217    )
 218    root_spec = (
 219        importlib.util.find_spec(root_name) if not root_path.exists()
 220        else importlib.util.spec_from_file_location(root_name, str(root_path))
 221    )
 222
 223    ### Check for updates before importing.
 224    _version = (
 225        determine_version(
 226            pathlib.Path(root_spec.origin),
 227            import_name=root_name, venv=venv, debug=debug
 228        ) if root_spec is not None and root_spec.origin is not None else None
 229    )
 230
 231    if _version is not None:
 232        if check_update:
 233            if need_update(
 234                None,
 235                import_name = root_name,
 236                version = _version,
 237                check_pypi = check_pypi,
 238                debug = debug,
 239            ):
 240                if install:
 241                    if not pip_install(
 242                        root_name,
 243                        venv = venv,
 244                        split = False,
 245                        check_update = check_update,
 246                        color = color,
 247                        debug = debug
 248                    ) and warn:
 249                        warn_function(
 250                            f"There's an update available for '{install_name}', "
 251                            + "but it failed to install. "
 252                            + "Try installig via Meerschaum with "
 253                            + "`install packages '{install_name}'`.",
 254                            ImportWarning,
 255                            stacklevel = 3,
 256                            color = False,
 257                        )
 258                elif warn:
 259                    warn_function(
 260                        f"There's an update available for '{root_name}'.",
 261                        stack = False,
 262                        color = False,
 263                    )
 264                spec = (
 265                    importlib.util.find_spec(import_name)
 266                    if mod_path is None or not mod_path.exists()
 267                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
 268                )
 269
 270
 271    if spec is None:
 272        try:
 273            mod = _import_module(import_name)
 274        except Exception as e:
 275            mod = None
 276        return mod
 277
 278    with Venv(venv, debug=debug):
 279        mod = importlib.util.module_from_spec(spec)
 280        old_sys_mod = sys.modules.get(import_name, None)
 281        sys.modules[import_name] = mod
 282
 283        try:
 284            with warnings.catch_warnings():
 285                warnings.filterwarnings('ignore', 'The NumPy')
 286                spec.loader.exec_module(mod)
 287        except Exception as e:
 288            pass
 289        mod = _import_module(import_name)
 290        if old_sys_mod is not None:
 291            sys.modules[import_name] = old_sys_mod
 292        else:
 293            del sys.modules[import_name]
 294    
 295    return mod
 296
 297
 298def _import_to_install_name(import_name: str, with_version: bool = True) -> str:
 299    """
 300    Try to translate an import name to an installation name.
 301    """
 302    install_name = all_packages.get(import_name, import_name)
 303    if with_version:
 304        return install_name
 305    return get_install_no_version(install_name)
 306
 307
 308def _import_to_dir_name(import_name: str) -> str:
 309    """
 310    Translate an import name to the package name in the sites-packages directory.
 311    """
 312    import re
 313    return re.split(
 314        r'[<>=\[]', all_packages.get(import_name, import_name)
 315    )[0].replace('-', '_').lower() 
 316
 317
 318def _install_to_import_name(install_name: str) -> str:
 319    """
 320    Translate an installation name to a package's import name.
 321    """
 322    _install_no_version = get_install_no_version(install_name)
 323    return get_install_names().get(_install_no_version, _install_no_version)
 324
 325
 326def get_install_no_version(install_name: str) -> str:
 327    """
 328    Strip the version information from the install name.
 329    """
 330    import re
 331    return re.split(r'[\[=<>,! \]]', install_name)[0]
 332
 333
 334import_versions = {}
 335def determine_version(
 336        path: pathlib.Path,
 337        import_name: Optional[str] = None,
 338        venv: Optional[str] = 'mrsm',
 339        search_for_metadata: bool = True,
 340        split: bool = True,
 341        warn: bool = False,
 342        debug: bool = False,
 343    ) -> Union[str, None]:
 344    """
 345    Determine a module's `__version__` string from its filepath.
 346    
 347    First it searches for pip metadata, then it attempts to import the module in a subprocess.
 348
 349    Parameters
 350    ----------
 351    path: pathlib.Path
 352        The file path of the module.
 353
 354    import_name: Optional[str], default None
 355        The name of the module. If omitted, it will be determined from the file path.
 356        Defaults to `None`.
 357
 358    venv: Optional[str], default 'mrsm'
 359        The virtual environment of the Python interpreter to use if importing is necessary.
 360
 361    search_for_metadata: bool, default True
 362        If `True`, search the pip site_packages directory (assumed to be the parent)
 363        for the corresponding dist-info directory.
 364
 365    warn: bool, default True
 366        If `True`, raise a warning if the module fails to import in the subprocess.
 367
 368    split: bool, default True
 369        If `True`, split the determined import name by periods to get the room name.
 370
 371    Returns
 372    -------
 373    The package's version string if available or `None`.
 374    If multiple versions are found, it will trigger an import in a subprocess.
 375
 376    """
 377    with _locks['import_versions']:
 378        if venv not in import_versions:
 379            import_versions[venv] = {}
 380    import importlib.metadata
 381    import re, os
 382    old_cwd = os.getcwd()
 383    if debug:
 384        from meerschaum.utils.debug import dprint
 385    from meerschaum.utils.warnings import warn as warn_function
 386    if import_name is None:
 387        import_name = path.parent.stem if path.stem == '__init__' else path.stem
 388        import_name = import_name.split('.')[0] if split else import_name
 389    if import_name in import_versions[venv]:
 390        return import_versions[venv][import_name]
 391    _version = None
 392    module_parent_dir = (
 393        path.parent.parent if path.stem == '__init__' else path.parent
 394    ) if path is not None else venv_target_path(venv, debug=debug)
 395
 396    installed_dir_name = _import_to_dir_name(import_name)
 397    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
 398
 399    ### First, check if a dist-info directory exists.
 400    _found_versions = []
 401    if search_for_metadata:
 402        for filename in os.listdir(module_parent_dir):
 403            if not filename.endswith('.dist-info'):
 404                continue
 405            filename_lower = filename.lower()
 406            if not filename_lower.startswith(clean_installed_dir_name + '-'):
 407                continue
 408            _v = filename.replace('.dist-info', '').split("-")[-1]
 409            _found_versions.append(_v)
 410
 411    if len(_found_versions) == 1:
 412        _version = _found_versions[0]
 413        with _locks['import_versions']:
 414            import_versions[venv][import_name] = _version
 415        return _found_versions[0]
 416
 417    if not _found_versions:
 418        try:
 419            import importlib.metadata as importlib_metadata
 420        except ImportError:
 421            importlib_metadata = attempt_import(
 422                'importlib_metadata',
 423                debug=debug, check_update=False, precheck=False,
 424                color=False, check_is_installed=False, lazy=False,
 425            )
 426        try:
 427            os.chdir(module_parent_dir)
 428            _version = importlib_metadata.metadata(import_name)['Version']
 429        except Exception as e:
 430            _version = None
 431        finally:
 432            os.chdir(old_cwd)
 433
 434        if _version is not None:
 435            with _locks['import_versions']:
 436                import_versions[venv][import_name] = _version
 437            return _version
 438
 439    if debug:
 440        print(f'Found multiple versions for {import_name}: {_found_versions}')
 441
 442    module_parent_dir_str = module_parent_dir.as_posix()
 443
 444    ### Not a pip package, so let's try importing the module directly (in a subprocess).
 445    _no_version_str = 'no-version'
 446    code = (
 447        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
 448        + f"module = importlib.import_module('{import_name}');\n"
 449        + "try:\n"
 450        + "  print(module.__version__ , end='')\n"
 451        + "except:\n"
 452        + f"  print('{_no_version_str}', end='')"
 453    )
 454    exit_code, stdout_bytes, stderr_bytes = venv_exec(
 455        code, venv=venv, with_extras=True, debug=debug
 456    )
 457    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
 458    _version = stdout.split('\n')[-1] if exit_code == 0 else None
 459    _version = _version if _version != _no_version_str else None
 460
 461    if _version is None:
 462        _version = _get_package_metadata(import_name, venv).get('version', None)
 463    if _version is None and warn:
 464        warn_function(
 465            f"Failed to determine a version for '{import_name}':\n{stderr}",
 466            stack = False
 467        )
 468
 469    ### If `__version__` doesn't exist, return `None`.
 470    import_versions[venv][import_name] = _version
 471    return _version
 472
 473
 474def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]:
 475    """
 476    Get a package's metadata from pip.
 477    This is useful for getting a version when no `__version__` is defined
 478    and multiple versions are installed.
 479
 480    Parameters
 481    ----------
 482    import_name: str
 483        The package's import or installation name.
 484
 485    venv: Optional[str]
 486        The virtual environment which contains the package.
 487
 488    Returns
 489    -------
 490    A dictionary of metadata from pip.
 491    """
 492    import re
 493    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 494    install_name = _import_to_install_name(import_name)
 495    _args = ['pip', 'show', install_name]
 496    if venv is not None:
 497        cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 498        _args += ['--cache-dir', cache_dir_path.as_posix()]
 499
 500    proc = run_python_package(
 501        'uv', _args,
 502        capture_output=True, as_proc=True, venv=venv, universal_newlines=True,
 503    )
 504    outs, errs = proc.communicate()
 505    lines = outs.split('\n')
 506    meta = {}
 507    for line in lines:
 508        vals = line.split(": ")
 509        if len(vals) != 2:
 510            continue
 511        k, v = vals[0].lower(), vals[1]
 512        if v and 'UNKNOWN' not in v:
 513            meta[k] = v
 514    return meta
 515
 516
 517def need_update(
 518        package: Optional['ModuleType'] = None,
 519        install_name: Optional[str] = None,
 520        import_name: Optional[str] = None,
 521        version: Optional[str] = None,
 522        check_pypi: bool = False,
 523        split: bool = True,
 524        color: bool = True,
 525        debug: bool = False,
 526        _run_determine_version: bool = True,
 527    ) -> bool:
 528    """
 529    Check if a Meerschaum dependency needs an update.
 530    Returns a bool for whether or not a package needs to be updated.
 531
 532    Parameters
 533    ----------
 534    package: 'ModuleType'
 535        The module of the package to be updated.
 536
 537    install_name: Optional[str], default None
 538        If provided, use this string to determine the required version.
 539        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
 540
 541    import_name:
 542        If provided, override the package's `__name__` string.
 543
 544    version: Optional[str], default None
 545        If specified, override the package's `__version__` string.
 546
 547    check_pypi: bool, default False
 548        If `True`, check pypi.org for updates.
 549        Defaults to `False`.
 550
 551    split: bool, default True
 552        If `True`, split the module's name on periods to detrive the root name.
 553        Defaults to `True`.
 554
 555    color: bool, default True
 556        If `True`, format debug output.
 557        Defaults to `True`.
 558
 559    debug: bool, default True
 560        Verbosity toggle.
 561
 562    Returns
 563    -------
 564    A bool indicating whether the package requires an update.
 565
 566    """
 567    if debug:
 568        from meerschaum.utils.debug import dprint
 569    from meerschaum.utils.warnings import warn as warn_function
 570    import re
 571    root_name = (
 572        package.__name__.split('.')[0] if split else package.__name__
 573    ) if import_name is None else (
 574        import_name.split('.')[0] if split else import_name
 575    )
 576    install_name = install_name or _import_to_install_name(root_name)
 577    with _locks['_checked_for_updates']:
 578        if install_name in _checked_for_updates:
 579            return False
 580        _checked_for_updates.add(install_name)
 581
 582    _install_no_version = get_install_no_version(install_name)
 583    required_version = install_name.replace(_install_no_version, '')
 584    if ']' in required_version:
 585        required_version = required_version.split(']')[1]
 586
 587    ### No minimum version was specified, and we're not going to check PyPI.
 588    if not required_version and not check_pypi:
 589        return False
 590
 591    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
 592    if '.dev' in required_version:
 593        required_version = required_version.split('.dev')[0]
 594    if version and '.dev' in version:
 595        version = version.split('.dev')[0]
 596
 597    try:
 598        if not version:
 599            if not _run_determine_version:
 600                version = determine_version(
 601                    pathlib.Path(package.__file__),
 602                    import_name=root_name, warn=False, debug=debug
 603                )
 604        if version is None:
 605            return False
 606    except Exception as e:
 607        if debug:
 608            dprint(str(e), color=color)
 609            dprint("No version could be determined from the installed package.", color=color)
 610        return False
 611    split_version = version.split('.')
 612    last_part = split_version[-1]
 613    if len(split_version) == 2:
 614        version = '.'.join(split_version) + '.0'
 615    elif 'dev' in last_part or 'rc' in last_part:
 616        tag = 'dev' if 'dev' in last_part else 'rc'
 617        last_sep = '-'
 618        if not last_part.startswith(tag):
 619            last_part = f'-{tag}'.join(last_part.split(tag))
 620            last_sep = '.'
 621        version = '.'.join(split_version[:-1]) + last_sep + last_part
 622    elif len(split_version) > 3:
 623        version = '.'.join(split_version[:3])
 624
 625    packaging_version = attempt_import(
 626        'packaging.version', check_update=False, lazy=False, debug=debug,
 627    )
 628
 629    ### Get semver if necessary
 630    if required_version:
 631        semver_path = get_module_path('semver', debug=debug)
 632        if semver_path is None:
 633            pip_install(_import_to_install_name('semver'), debug=debug)
 634        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
 635    if check_pypi:
 636        ### Check PyPI for updates
 637        update_checker = attempt_import(
 638            'update_checker', lazy=False, check_update=False, debug=debug
 639        )
 640        checker = update_checker.UpdateChecker()
 641        result = checker.check(_install_no_version, version)
 642    else:
 643        ### Skip PyPI and assume we can't be sure.
 644        result = None
 645
 646    ### Compare PyPI's version with our own.
 647    if result is not None:
 648        ### We have a result from PyPI and a stated required version.
 649        if required_version:
 650            try:
 651                return semver.Version.parse(result.available_version).match(required_version)
 652            except AttributeError as e:
 653                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 654                semver = manually_import_module('semver', venv='mrsm')
 655                return semver.Version.parse(version).match(required_version)
 656            except Exception as e:
 657                if debug:
 658                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
 659                return False
 660
 661        ### If `check_pypi` and we don't have a required version, check if PyPI's version
 662        ### is newer than the installed version.
 663        else:
 664            return (
 665                packaging_version.parse(result.available_version) > 
 666                packaging_version.parse(version)
 667            )
 668
 669    ### We might be depending on a prerelease.
 670    ### Sanity check that the required version is not greater than the installed version. 
 671    required_version = (
 672        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
 673        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
 674        .replace('-py3-none-any.whl', '')
 675    )
 676
 677    if 'a' in required_version:
 678        required_version = required_version.replace('a', '-dev').replace('+mrsm', '')
 679        version = version.replace('a', '-dev').replace('+mrsm', '')
 680    try:
 681        return (
 682            (not semver.Version.parse(version).match(required_version))
 683            if required_version else False
 684        )
 685    except AttributeError as e:
 686        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 687        semver = manually_import_module('semver', venv='mrsm', debug=debug)
 688        return (
 689            (not semver.Version.parse(version).match(required_version))
 690            if required_version else False
 691        )
 692    except Exception as e:
 693        print(f"Unable to parse version ({version}) for package '{import_name}'.")
 694        print(e)
 695        if debug:
 696            dprint(e)
 697        return False
 698    try:
 699        return (
 700            packaging_version.parse(version) > 
 701            packaging_version.parse(required_version)
 702        )
 703    except Exception as e:
 704        if debug:
 705            dprint(e)
 706        return False
 707    return False
 708
 709
 710def get_pip(
 711        venv: Optional[str] = 'mrsm',
 712        color: bool = True,
 713        debug: bool = False,
 714    ) -> bool:
 715    """
 716    Download and run the get-pip.py script.
 717
 718    Parameters
 719    ----------
 720    venv: Optional[str], default 'mrsm'
 721        The virtual environment into which to install `pip`.
 722
 723    color: bool, default True
 724        If `True`, force color output.
 725
 726    debug: bool, default False
 727        Verbosity toggle.
 728
 729    Returns
 730    -------
 731    A bool indicating success.
 732
 733    """
 734    import sys, subprocess
 735    from meerschaum.utils.misc import wget
 736    from meerschaum.config._paths import CACHE_RESOURCES_PATH
 737    from meerschaum.config.static import STATIC_CONFIG
 738    url = STATIC_CONFIG['system']['urls']['get-pip.py']
 739    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
 740    try:
 741        wget(url, dest, color=False, debug=debug)
 742    except Exception as e:
 743        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
 744        sys.exit(1)
 745    if venv is not None:
 746        init_venv(venv=venv, debug=debug)
 747    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
 748    return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0
 749
 750
 751def pip_install(
 752        *install_names: str,
 753        args: Optional[List[str]] = None,
 754        requirements_file_path: Union[pathlib.Path, str, None] = None,
 755        venv: Optional[str] = 'mrsm',
 756        split: bool = False,
 757        check_update: bool = True,
 758        check_pypi: bool = True,
 759        check_wheel: bool = True,
 760        _uninstall: bool = False,
 761        _from_completely_uninstall: bool = False,
 762        _install_uv_pip: bool = True,
 763        color: bool = True,
 764        silent: bool = False,
 765        debug: bool = False,
 766    ) -> bool:
 767    """
 768    Install packages from PyPI with `pip`.
 769
 770    Parameters
 771    ----------
 772    *install_names: str
 773        The installation names of packages to be installed.
 774        This includes version restrictions.
 775        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 776        from its import name.
 777        
 778    args: Optional[List[str]], default None
 779        A list of command line arguments to pass to `pip`.
 780        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 781
 782    requirements_file_path: Optional[pathlib.Path, str], default None
 783        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 784
 785    venv: str, default 'mrsm'
 786        The virtual environment to install into.
 787
 788    split: bool, default False
 789        If `True`, split on periods and only install the root package name.
 790
 791    check_update: bool, default True
 792        If `True`, check if the package requires an update.
 793
 794    check_pypi: bool, default True
 795        If `True` and `check_update` is `True`, check PyPI for the latest version.
 796
 797    check_wheel: bool, default True
 798        If `True`, check if `wheel` is available.
 799
 800    _uninstall: bool, default False
 801        If `True`, uninstall packages instead.
 802
 803    color: bool, default True
 804        If `True`, include color in debug text.
 805
 806    silent: bool, default False
 807        If `True`, skip printing messages.
 808
 809    debug: bool, default False
 810        Verbosity toggle.
 811
 812    Returns
 813    -------
 814    A bool indicating success.
 815
 816    """
 817    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 818    from meerschaum.config import get_config
 819    from meerschaum.config.static import STATIC_CONFIG
 820    from meerschaum.utils.warnings import warn
 821    from meerschaum.utils.misc import is_android
 822    if args is None:
 823        args = ['--upgrade'] if not _uninstall else []
 824    if color:
 825        ANSI, UNICODE = True, True
 826    else:
 827        ANSI, UNICODE = False, False
 828    if check_wheel:
 829        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 830
 831    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
 832    inside_daemon = daemon_env_var in os.environ
 833    if inside_daemon:
 834        silent = True
 835
 836    _args = list(args)
 837    have_pip = venv_contains_package('pip', venv=None, debug=debug)
 838    try:
 839        import pip
 840        have_pip = True
 841    except ImportError:
 842        have_pip = False
 843    try:
 844        import uv
 845        uv_bin = uv.find_uv_bin()
 846        have_uv_pip = True
 847    except (ImportError, FileNotFoundError):
 848        uv_bin = None
 849        have_uv_pip = False
 850    if have_pip and not have_uv_pip and _install_uv_pip and not is_android():
 851        if not pip_install(
 852            'uv',
 853            venv=None,
 854            debug=debug,
 855            _install_uv_pip=False,
 856            check_update=False,
 857            check_pypi=False,
 858            check_wheel=False,
 859        ) and not silent:
 860            warn(
 861                f"Failed to install `uv` for virtual environment '{venv}'.",
 862                color=False,
 863            )
 864
 865    use_uv_pip = (
 866        venv_contains_package('uv', venv=None, debug=debug)
 867        and uv_bin is not None
 868        and venv is not None
 869    )
 870
 871    import sys
 872    if not have_pip and not use_uv_pip:
 873        if not get_pip(venv=venv, color=color, debug=debug):
 874            import sys
 875            minor = sys.version_info.minor
 876            print(
 877                "\nFailed to import `pip` and `ensurepip`.\n"
 878                + "If you are running Ubuntu/Debian, "
 879                + "you might need to install `python3.{minor}-distutils`:\n\n"
 880                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
 881                + "Please install pip and restart Meerschaum.\n\n"
 882                + "You can find instructions on installing `pip` here:\n"
 883                + "https://pip.pypa.io/en/stable/installing/"
 884            )
 885            sys.exit(1)
 886    
 887    with Venv(venv, debug=debug):
 888        if venv is not None:
 889            if (
 890                '--ignore-installed' not in args
 891                and '-I' not in _args
 892                and not _uninstall
 893                and not use_uv_pip
 894            ):
 895                _args += ['--ignore-installed']
 896            if '--cache-dir' not in args and not _uninstall:
 897                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 898                _args += ['--cache-dir', str(cache_dir_path)]
 899
 900        if 'pip' not in ' '.join(_args) and not use_uv_pip:
 901            if check_update and not _uninstall:
 902                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
 903                if need_update(pip, check_pypi=check_pypi, debug=debug):
 904                    _args.append(all_packages['pip'])
 905
 906        _args = (['install'] if not _uninstall else ['uninstall']) + _args
 907
 908        if check_wheel and not _uninstall and not use_uv_pip:
 909            if not have_wheel:
 910                setup_packages_to_install = (
 911                    ['setuptools', 'wheel']
 912                    + ([] if is_android() else ['uv'])
 913                )
 914                if not pip_install(
 915                    *setup_packages_to_install,
 916                    venv=venv,
 917                    check_update=False,
 918                    check_pypi=False,
 919                    check_wheel=False,
 920                    debug=debug,
 921                    _install_uv_pip=False,
 922                ) and not silent:
 923                    from meerschaum.utils.misc import items_str
 924                    warn(
 925                        (
 926                            f"Failed to install {items_str(setup_packages_to_install)} for virtual "
 927                            + f"environment '{venv}'."
 928                        ),
 929                        color=False,
 930                    )
 931
 932        if requirements_file_path is not None:
 933            _args.append('-r')
 934            _args.append(pathlib.Path(requirements_file_path).resolve().as_posix())
 935
 936        if not ANSI and '--no-color' not in _args:
 937            _args.append('--no-color')
 938
 939        if '--no-input' not in _args and not use_uv_pip:
 940            _args.append('--no-input')
 941
 942        if _uninstall and '-y' not in _args and not use_uv_pip:
 943            _args.append('-y')
 944
 945        if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip:
 946            _args.append('--no-warn-conflicts')
 947
 948        if '--disable-pip-version-check' not in _args and not use_uv_pip:
 949            _args.append('--disable-pip-version-check')
 950
 951        if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
 952            if venv is not None:
 953                _args += ['--target', venv_target_path(venv, debug=debug)]
 954        elif (
 955            '--target' not in _args
 956                and '-t' not in _args
 957                and not inside_venv()
 958                and not _uninstall
 959                and not use_uv_pip
 960        ):
 961            _args += ['--user']
 962
 963        if debug:
 964            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
 965                if use_uv_pip:
 966                    _args.append('--verbose')
 967        else:
 968            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
 969                pass
 970
 971        _packages = [
 972            (install_name if not _uninstall else get_install_no_version(install_name))
 973            for install_name in install_names
 974        ]
 975        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
 976        for p in _packages:
 977            msg += f'\n  - {p}'
 978        if not silent:
 979            print(msg)
 980
 981        if _uninstall and not _from_completely_uninstall and not use_uv_pip:
 982            for install_name in _packages:
 983                _install_no_version = get_install_no_version(install_name)
 984                if _install_no_version in ('pip', 'wheel', 'uv'):
 985                    continue
 986                if not completely_uninstall_package(
 987                    _install_no_version,
 988                    venv=venv, debug=debug,
 989                ) and not silent:
 990                    warn(
 991                        f"Failed to clean up package '{_install_no_version}'.",
 992                    )
 993
 994        ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease.
 995        if use_uv_pip:
 996            _args.insert(0, 'pip')
 997            if not _uninstall and get_prerelease_dependencies(_packages):
 998                _args.append('--prerelease=allow')
 999
1000        rc = run_python_package(
1001            ('pip' if not use_uv_pip else 'uv'),
1002            _args + _packages,
1003            venv=None,
1004            env=_get_pip_os_env(color=color),
1005            debug=debug,
1006        )
1007        if debug:
1008            print(f"{rc=}")
1009        success = rc == 0
1010
1011    msg = (
1012        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
1013        else "Failed to " + ('un' if _uninstall else '') + "install packages."
1014    )
1015    if not silent:
1016        print(msg)
1017    if debug and not silent:
1018        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
1019    return success
1020
1021
1022def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1023    """
1024    Return a list of explicitly prerelease dependencies from a list of packages.
1025    """
1026    if _packages is None:
1027        _packages = list(all_packages.keys())
1028    prelrease_strings = ['dev', 'rc', 'a']
1029    prerelease_packages = []
1030    for install_name in _packages:
1031        _install_no_version = get_install_no_version(install_name)
1032        import_name = _install_to_import_name(install_name)
1033        install_with_version = _import_to_install_name(import_name)
1034        version_only = (
1035            install_with_version.lower().replace(_install_no_version.lower(), '')
1036            .split(']')[-1]
1037        )
1038
1039        is_prerelease = False
1040        for prelrease_string in prelrease_strings:
1041            if prelrease_string in version_only:
1042                is_prerelease = True
1043
1044        if is_prerelease:
1045            prerelease_packages.append(install_name)
1046    return prerelease_packages
1047
1048
1049def completely_uninstall_package(
1050        install_name: str,
1051        venv: str = 'mrsm',
1052        debug: bool = False,
1053    ) -> bool:
1054    """
1055    Continue calling `pip uninstall` until a package is completely
1056    removed from a virtual environment. 
1057    This is useful for dealing with multiple installed versions of a package.
1058    """
1059    attempts = 0
1060    _install_no_version = get_install_no_version(install_name)
1061    clean_install_no_version = _install_no_version.lower().replace('-', '_')
1062    installed_versions = []
1063    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1064    if not vtp.exists():
1065        return True
1066
1067    for file_name in os.listdir(vtp):
1068        if not file_name.endswith('.dist-info'):
1069            continue
1070        clean_dist_info = file_name.replace('-', '_').lower()
1071        if not clean_dist_info.startswith(clean_install_no_version):
1072            continue
1073        installed_versions.append(file_name)
1074
1075    max_attempts = len(installed_versions)
1076    while attempts < max_attempts:
1077        if not venv_contains_package(
1078            _install_to_import_name(_install_no_version),
1079            venv=venv, debug=debug,
1080        ):
1081            return True
1082        if not pip_uninstall(
1083            _install_no_version,
1084            venv = venv,
1085            silent = (not debug),
1086            _from_completely_uninstall = True,
1087            debug = debug,
1088        ):
1089            return False
1090        attempts += 1
1091    return False
1092
1093
1094def pip_uninstall(
1095        *args, **kw
1096    ) -> bool:
1097    """
1098    Uninstall Python packages.
1099    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
1100    """
1101    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
1102
1103
1104def run_python_package(
1105        package_name: str,
1106        args: Optional[List[str]] = None,
1107        venv: Optional[str] = 'mrsm',
1108        cwd: Optional[str] = None,
1109        foreground: bool = False,
1110        as_proc: bool = False,
1111        capture_output: bool = False,
1112        debug: bool = False,
1113        **kw: Any,
1114    ) -> Union[int, subprocess.Popen, None]:
1115    """
1116    Runs an installed python package.
1117    E.g. Translates to `/usr/bin/python -m [package]`
1118
1119    Parameters
1120    ----------
1121    package_name: str
1122        The Python module to be executed.
1123
1124    args: Optional[List[str]], default None
1125        Additional command line arguments to be appended after `-m [package]`.
1126
1127    venv: Optional[str], default 'mrsm'
1128        If specified, execute the Python interpreter from a virtual environment.
1129
1130    cwd: Optional[str], default None
1131        If specified, change directories before starting the process.
1132        Defaults to `None`.
1133
1134    as_proc: bool, default False
1135        If `True`, return a `subprocess.Popen` object.
1136
1137    capture_output: bool, default False
1138        If `as_proc` is `True`, capture stdout and stderr.
1139
1140    foreground: bool, default False
1141        If `True`, start the subprocess as a foreground process.
1142        Defaults to `False`.
1143
1144    kw: Any
1145        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1146        and by extension `subprocess.Popen()`.
1147
1148    Returns
1149    -------
1150    Either a return code integer or a `subprocess.Popen` object
1151    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1152    """
1153    import sys, platform
1154    import subprocess
1155    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1156    from meerschaum.utils.process import run_process
1157    from meerschaum.utils.warnings import warn
1158    if args is None:
1159        args = []
1160    old_cwd = os.getcwd()
1161    if cwd is not None:
1162        os.chdir(cwd)
1163    executable = venv_executable(venv=venv)
1164    venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
1165    env_dict = kw.get('env', os.environ).copy()
1166    if venv_path is not None:
1167        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
1168    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1169    import traceback
1170    if debug:
1171        print(command, file=sys.stderr)
1172    try:
1173        to_return = run_process(
1174            command,
1175            foreground = foreground,
1176            as_proc = as_proc,
1177            capture_output = capture_output,
1178            **kw
1179        )
1180    except Exception as e:
1181        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1182        warn(msg, color=False)
1183        stdout, stderr = (
1184            (None, None)
1185            if not capture_output
1186            else (subprocess.PIPE, subprocess.PIPE)
1187        )
1188        proc = subprocess.Popen(
1189            command,
1190            stdout = stdout,
1191            stderr = stderr,
1192            env = env_dict,
1193        )
1194        to_return = proc if as_proc else proc.wait()
1195    except KeyboardInterrupt:
1196        to_return = 1 if not as_proc else None
1197    os.chdir(old_cwd)
1198    return to_return
1199
1200
1201def attempt_import(
1202        *names: str,
1203        lazy: bool = True,
1204        warn: bool = True,
1205        install: bool = True,
1206        venv: Optional[str] = 'mrsm',
1207        precheck: bool = True,
1208        split: bool = True,
1209        check_update: bool = False,
1210        check_pypi: bool = False,
1211        check_is_installed: bool = True,
1212        allow_outside_venv: bool = True,
1213        color: bool = True,
1214        debug: bool = False
1215    ) -> Any:
1216    """
1217    Raise a warning if packages are not installed; otherwise import and return modules.
1218    If `lazy` is `True`, return lazy-imported modules.
1219    
1220    Returns tuple of modules if multiple names are provided, else returns one module.
1221    
1222    Parameters
1223    ----------
1224    names: List[str]
1225        The packages to be imported.
1226
1227    lazy: bool, default True
1228        If `True`, lazily load packages.
1229
1230    warn: bool, default True
1231        If `True`, raise a warning if a package cannot be imported.
1232
1233    install: bool, default True
1234        If `True`, attempt to install a missing package into the designated virtual environment.
1235        If `check_update` is True, install updates if available.
1236
1237    venv: Optional[str], default 'mrsm'
1238        The virtual environment in which to search for packages and to install packages into.
1239
1240    precheck: bool, default True
1241        If `True`, attempt to find module before importing (necessary for checking if modules exist
1242        and retaining lazy imports), otherwise assume lazy is `False`.
1243
1244    split: bool, default True
1245        If `True`, split packages' names on `'.'`.
1246
1247    check_update: bool, default False
1248        If `True` and `install` is `True`, install updates if the required minimum version
1249        does not match.
1250
1251    check_pypi: bool, default False
1252        If `True` and `check_update` is `True`, check PyPI when determining whether
1253        an update is required.
1254
1255    check_is_installed: bool, default True
1256        If `True`, check if the package is contained in the virtual environment.
1257
1258    allow_outside_venv: bool, default True
1259        If `True`, search outside of the specified virtual environment
1260        if the package cannot be found.
1261        Setting to `False` will reinstall the package into a virtual environment, even if it
1262        is installed outside.
1263
1264    color: bool, default True
1265        If `False`, do not print ANSI colors.
1266
1267    Returns
1268    -------
1269    The specified modules. If they're not available and `install` is `True`, it will first
1270    download them into a virtual environment and return the modules.
1271
1272    Examples
1273    --------
1274    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1275    >>> pandas = attempt_import('pandas')
1276
1277    """
1278
1279    import importlib.util
1280
1281    ### to prevent recursion, check if parent Meerschaum package is being imported
1282    if names == ('meerschaum',):
1283        return _import_module('meerschaum')
1284
1285    if venv == 'mrsm' and _import_hook_venv is not None:
1286        if debug:
1287            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1288        venv = _import_hook_venv
1289
1290    _warnings = _import_module('meerschaum.utils.warnings')
1291    warn_function = _warnings.warn
1292
1293    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1294        with Venv(venv=venv, debug=debug):
1295            ### determine the import method (lazy vs normal)
1296            from meerschaum.utils.misc import filter_keywords
1297            import_method = (
1298                _import_module if not lazy
1299                else lazy_import
1300            )
1301            try:
1302                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1303            except Exception as e:
1304                if warn:
1305                    import traceback
1306                    traceback.print_exception(type(e), e, e.__traceback__)
1307                    warn_function(
1308                        f"Failed to import module '{_name}'.\nException:\n{e}",
1309                        ImportWarning,
1310                        stacklevel = (5 if lazy else 4),
1311                        color = False,
1312                    )
1313                mod = None
1314        return mod
1315
1316    modules = []
1317    for name in names:
1318        ### Check if package is a declared dependency.
1319        root_name = name.split('.')[0] if split else name
1320        install_name = _import_to_install_name(root_name)
1321
1322        if install_name is None:
1323            install_name = root_name
1324            if warn and root_name != 'plugins':
1325                warn_function(
1326                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1327                    ImportWarning,
1328                    stacklevel = 3,
1329                    color = False
1330                )
1331
1332        ### Determine if the package exists.
1333        if precheck is False:
1334            found_module = (
1335                do_import(
1336                    name, debug=debug, warn=False, venv=venv, color=color,
1337                    check_update=False, check_pypi=False, split=split,
1338                ) is not None
1339            )
1340        else:
1341            if check_is_installed:
1342                with _locks['_is_installed_first_check']:
1343                    if not _is_installed_first_check.get(name, False):
1344                        package_is_installed = is_installed(
1345                            name,
1346                            venv = venv,
1347                            split = split,
1348                            allow_outside_venv = allow_outside_venv,
1349                            debug = debug,
1350                        )
1351                        _is_installed_first_check[name] = package_is_installed
1352                    else:
1353                        package_is_installed = _is_installed_first_check[name]
1354            else:
1355                package_is_installed = _is_installed_first_check.get(
1356                    name,
1357                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1358                )
1359            found_module = package_is_installed
1360
1361        if not found_module:
1362            if install:
1363                if not pip_install(
1364                    install_name,
1365                    venv = venv,
1366                    split = False,
1367                    check_update = check_update,
1368                    color = color,
1369                    debug = debug
1370                ) and warn:
1371                    warn_function(
1372                        f"Failed to install '{install_name}'.",
1373                        ImportWarning,
1374                        stacklevel = 3,
1375                        color = False,
1376                    )
1377            elif warn:
1378                ### Raise a warning if we can't find the package and install = False.
1379                warn_function(
1380                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1381                     + "some features will not work correctly."
1382                     + f"\n\nSet install=True when calling attempt_import.\n"),
1383                    ImportWarning,
1384                    stacklevel = 3,
1385                    color = False,
1386                )
1387
1388        ### Do the import. Will be lazy if lazy=True.
1389        m = do_import(
1390            name, debug=debug, warn=warn, venv=venv, color=color,
1391            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1392        )
1393        modules.append(m)
1394
1395    modules = tuple(modules)
1396    if len(modules) == 1:
1397        return modules[0]
1398    return modules
1399
1400
1401def lazy_import(
1402        name: str,
1403        local_name: str = None,
1404        **kw
1405    ) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1406    """
1407    Lazily import a package.
1408    """
1409    from meerschaum.utils.packages.lazy_loader import LazyLoader
1410    if local_name is None:
1411        local_name = name
1412    return LazyLoader(
1413        local_name,
1414        globals(),
1415        name,
1416        **kw
1417    )
1418
1419
1420def pandas_name() -> str:
1421    """
1422    Return the configured name for `pandas`.
1423    
1424    Below are the expected possible values:
1425
1426    - 'pandas'
1427    - 'modin.pandas'
1428    - 'dask.dataframe'
1429
1430    """
1431    from meerschaum.config import get_config
1432    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1433    if pandas_module_name == 'modin':
1434        pandas_module_name = 'modin.pandas'
1435    elif pandas_module_name == 'dask':
1436        pandas_module_name = 'dask.dataframe'
1437
1438    return pandas_module_name
1439
1440
1441emitted_pandas_warning: bool = False
1442def import_pandas(
1443        debug: bool = False,
1444        lazy: bool = False,
1445        **kw
1446    ) -> 'ModuleType':
1447    """
1448    Quality-of-life function to attempt to import the configured version of `pandas`.
1449    """
1450    import sys
1451    pandas_module_name = pandas_name()
1452    global emitted_pandas_warning
1453
1454    if pandas_module_name != 'pandas':
1455        with _locks['emitted_pandas_warning']:
1456            if not emitted_pandas_warning:
1457                from meerschaum.utils.warnings import warn
1458                emitted_pandas_warning = True
1459                warn(
1460                    (
1461                        "You are using an alternative Pandas implementation "
1462                        + f"'{pandas_module_name}'"
1463                        + "\n   Features may not work as expected."
1464                    ),
1465                    stack = False,
1466                )
1467
1468    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1469    pandas = attempt_import('pandas', debug=debug, lazy=False, **kw)
1470    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1471    return pd
1472
1473
1474def import_rich(
1475        lazy: bool = True,
1476        debug: bool = False,
1477        **kw : Any
1478    ) -> 'ModuleType':
1479    """
1480    Quality of life function for importing `rich`.
1481    """
1482    from meerschaum.utils.formatting import ANSI, UNICODE
1483    if not ANSI and not UNICODE:
1484        return None
1485
1486    ## need typing_extensions for `from rich import box`
1487    typing_extensions = attempt_import(
1488        'typing_extensions', lazy=False, debug=debug
1489    )
1490    pygments = attempt_import(
1491        'pygments', lazy=False,
1492    )
1493    rich = attempt_import(
1494        'rich', lazy=lazy,
1495        **kw
1496    )
1497    return rich
1498
1499
1500def _dash_less_than_2(**kw) -> bool:
1501    dash = attempt_import('dash', **kw)
1502    if dash is None:
1503        return None
1504    packaging_version = attempt_import('packaging.version', **kw)
1505    return (
1506        packaging_version.parse(dash.__version__) < 
1507        packaging_version.parse('2.0.0')
1508    )
1509
1510
1511def import_dcc(warn=False, **kw) -> 'ModuleType':
1512    """
1513    Import Dash Core Components (`dcc`).
1514    """
1515    return (
1516        attempt_import('dash_core_components', warn=warn, **kw)
1517        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1518    )
1519
1520
1521def import_html(warn=False, **kw) -> 'ModuleType':
1522    """
1523    Import Dash HTML Components (`html`).
1524    """
1525    return (
1526        attempt_import('dash_html_components', warn=warn, **kw)
1527        if _dash_less_than_2(warn=warn, **kw)
1528        else attempt_import('dash.html', warn=warn, **kw)
1529    )
1530
1531
1532def get_modules_from_package(
1533        package: 'package',
1534        names: bool = False,
1535        recursive: bool = False,
1536        lazy: bool = False,
1537        modules_venvs: bool = False,
1538        debug: bool = False
1539    ):
1540    """
1541    Find and import all modules in a package.
1542    
1543    Returns
1544    -------
1545    Either list of modules or tuple of lists.
1546    """
1547    from os.path import dirname, join, isfile, isdir, basename
1548    import glob
1549
1550    pattern = '*' if recursive else '*.py'
1551    package_path = dirname(package.__file__ or package.__path__[0])
1552    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1553    _all = [
1554        basename(f)[:-3] if isfile(f) else basename(f)
1555        for f in module_names
1556            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1557               and not f.endswith('__init__.py')
1558               and not f.endswith('__pycache__')
1559    ]
1560
1561    if debug:
1562        from meerschaum.utils.debug import dprint
1563        dprint(str(_all))
1564    modules = []
1565    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1566        ### there's probably a better way than a try: catch but it'll do for now
1567        try:
1568            ### if specified, activate the module's virtual environment before importing.
1569            ### NOTE: this only considers the filename, so two modules from different packages
1570            ### may end up sharing virtual environments.
1571            if modules_venvs:
1572                activate_venv(module_name.split('.')[-1], debug=debug)
1573            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1574            modules.append(m)
1575        except Exception as e:
1576            if debug:
1577                dprint(str(e))
1578        finally:
1579            if modules_venvs:
1580                deactivate_venv(module_name.split('.')[-1], debug=debug)
1581    if names:
1582        return _all, modules
1583
1584    return modules
1585
1586
1587def import_children(
1588        package: Optional['ModuleType'] = None,
1589        package_name: Optional[str] = None,
1590        types : Optional[List[str]] = None,
1591        lazy: bool = True,
1592        recursive: bool = False,
1593        debug: bool = False
1594    ) -> List['ModuleType']:
1595    """
1596    Import all functions in a package to its `__init__`.
1597
1598    Parameters
1599    ----------
1600    package: Optional[ModuleType], default None
1601        Package to import its functions into.
1602        If `None` (default), use parent.
1603
1604    package_name: Optional[str], default None
1605        Name of package to import its functions into
1606        If None (default), use parent.
1607
1608    types: Optional[List[str]], default None
1609        Types of members to return.
1610        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1611
1612    Returns
1613    -------
1614    A list of modules.
1615    """
1616    import sys, inspect
1617
1618    if types is None:
1619        types = ['method', 'builtin', 'function', 'class', 'module']
1620
1621    ### if package_name and package are None, use parent
1622    if package is None and package_name is None:
1623        package_name = inspect.stack()[1][0].f_globals['__name__']
1624
1625    ### populate package or package_name from other other
1626    if package is None:
1627        package = sys.modules[package_name]
1628    elif package_name is None:
1629        package_name = package.__name__
1630
1631    ### Set attributes in sys module version of package.
1632    ### Kinda like setting a dictionary
1633    ###   functions[name] = func
1634    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1635    _all, members = [], []
1636    objects = []
1637    for module in modules:
1638        _objects = []
1639        for ob in inspect.getmembers(module):
1640            for t in types:
1641                ### ob is a tuple of (name, object)
1642                if getattr(inspect, 'is' + t)(ob[1]):
1643                    _objects.append(ob)
1644
1645        if 'module' in types:
1646            _objects.append((module.__name__.split('.')[0], module))
1647        objects += _objects
1648    for ob in objects:
1649        setattr(sys.modules[package_name], ob[0], ob[1])
1650        _all.append(ob[0])
1651        members.append(ob[1])
1652
1653    if debug:
1654        from meerschaum.utils.debug import dprint
1655        dprint(str(_all))
1656    ### set __all__ for import *
1657    setattr(sys.modules[package_name], '__all__', _all)
1658    return members
1659
1660
1661_reload_module_cache = {}
1662def reload_package(
1663        package: str,
1664        skip_submodules: Optional[List[str]] = None,
1665        lazy: bool = False,
1666        debug: bool = False,
1667        **kw: Any
1668    ):
1669    """
1670    Recursively load a package's subpackages, even if they were not previously loaded.
1671    """
1672    import sys
1673    if isinstance(package, str):
1674        package_name = package
1675    else:
1676        try:
1677            package_name = package.__name__
1678        except Exception as e:
1679            package_name = str(package)
1680
1681    skip_submodules = skip_submodules or []
1682    if 'meerschaum.utils.packages' not in skip_submodules:
1683        skip_submodules.append('meerschaum.utils.packages')
1684    def safeimport():
1685        subs = [
1686            m for m in sys.modules
1687            if m.startswith(package_name + '.')
1688        ]
1689        subs_to_skip = []
1690        for skip_mod in skip_submodules:
1691            for mod in subs:
1692                if mod.startswith(skip_mod):
1693                    subs_to_skip.append(mod)
1694                    continue
1695
1696        subs = [m for m in subs if m not in subs_to_skip]
1697        for module_name in subs:
1698            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1699        if not subs_to_skip:
1700            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1701
1702        return _import_module(package_name)
1703
1704    return safeimport()
1705
1706
1707def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1708    """
1709    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1710    """
1711    reload_package(
1712        'meerschaum',
1713        skip_submodules = [
1714            'meerschaum._internal.shell',
1715            'meerschaum.utils.pool',
1716        ]
1717    )
1718
1719    from meerschaum.plugins import reload_plugins
1720    from meerschaum._internal.shell.Shell import _insert_shell_actions
1721    reload_plugins(debug=debug)
1722    _insert_shell_actions()
1723    return True, "Success"
1724
1725
1726def is_installed(
1727        import_name: str,
1728        venv: Optional[str] = 'mrsm',
1729        split: bool = True,
1730        allow_outside_venv: bool = True,
1731        debug: bool = False,
1732    ) -> bool:
1733    """
1734    Check whether a package is installed.
1735
1736    Parameters
1737    ----------
1738    import_name: str
1739        The import name of the module.
1740
1741    venv: Optional[str], default 'mrsm'
1742        The venv in which to search for the module.
1743
1744    split: bool, default True
1745        If `True`, split on periods to determine the root module name.
1746
1747    allow_outside_venv: bool, default True
1748        If `True`, search outside of the specified virtual environment
1749        if the package cannot be found.
1750    """
1751    if debug:
1752        from meerschaum.utils.debug import dprint
1753    root_name = import_name.split('.')[0] if split else import_name
1754    import importlib.util
1755    with Venv(venv, debug=debug):
1756        try:
1757            spec_path = pathlib.Path(
1758                get_module_path(root_name, venv=venv, debug=debug)
1759                or
1760                (
1761                    importlib.util.find_spec(root_name).origin 
1762                    if venv is not None and allow_outside_venv
1763                    else None
1764                )
1765            )
1766        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1767            spec_path = None
1768
1769        found = (
1770            not need_update(
1771                None, import_name = root_name,
1772                _run_determine_version = False,
1773                check_pypi = False,
1774                version = determine_version(
1775                    spec_path, venv=venv, debug=debug, import_name=root_name
1776                ),
1777                debug = debug,
1778            )
1779        ) if spec_path is not None else False
1780
1781    return found
1782
1783
1784def venv_contains_package(
1785        import_name: str,
1786        venv: Optional[str] = 'mrsm',
1787        split: bool = True,
1788        debug: bool = False,
1789    ) -> bool:
1790    """
1791    Search the contents of a virtual environment for a package.
1792    """
1793    import site
1794    import pathlib
1795    root_name = import_name.split('.')[0] if split else import_name
1796    return get_module_path(root_name, venv=venv, debug=debug) is not None
1797
1798
1799def package_venv(package: 'ModuleType') -> Union[str, None]:
1800    """
1801    Inspect a package and return the virtual environment in which it presides.
1802    """
1803    import os
1804    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1805    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1806        return None
1807    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
1808
1809
1810def ensure_readline() -> 'ModuleType':
1811    """Make sure that the `readline` package is able to be imported."""
1812    import sys
1813    try:
1814        import readline
1815    except ImportError:
1816        readline = None
1817
1818    if readline is None:
1819        import platform
1820        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1821        try:
1822            rl = attempt_import(
1823                rl_name,
1824                lazy = False,
1825                install = True,
1826                venv = None,
1827                warn = False,
1828            )
1829        except (ImportError, ModuleNotFoundError):
1830            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1831                print(f"Unable to import {rl_name}!", file=sys.stderr)
1832                sys.exit(1)
1833
1834    sys.modules['readline'] = readline
1835    return readline
1836
1837_pkg_resources_get_distribution = None
1838_custom_distributions = {}
1839def _monkey_patch_get_distribution(_dist: str, _version: str) -> None:
1840    """
1841    Monkey patch `pkg_resources.get_distribution` to allow for importing `flask_compress`.
1842    """
1843    import pkg_resources
1844    from collections import namedtuple
1845    global _pkg_resources_get_distribution
1846    with _locks['_pkg_resources_get_distribution']:
1847        _pkg_resources_get_distribution = pkg_resources.get_distribution
1848    _custom_distributions[_dist] = _version
1849    _Dist = namedtuple('_Dist', ['version'])
1850    def _get_distribution(dist):
1851        """Hack for flask-compress."""
1852        if dist in _custom_distributions:
1853            return _Dist(_custom_distributions[dist])
1854        return _pkg_resources_get_distribution(dist)
1855    pkg_resources.get_distribution = _get_distribution
1856
1857
1858def _get_pip_os_env(color: bool = True):
1859    """
1860    Return the environment variables context in which `pip` should be run.
1861    See PEP 668 for why we are overriding the environment.
1862    """
1863    import os, sys, platform
1864    python_bin_path = pathlib.Path(sys.executable)
1865    pip_os_env = os.environ.copy()
1866    path_str = pip_os_env.get('PATH', '') or ''
1867    path_sep = ':' if platform.system() != 'Windows' else ';'
1868    pip_os_env.update({
1869        'PIP_BREAK_SYSTEM_PACKAGES': 'true',
1870        ('FORCE_COLOR' if color else 'NO_COLOR'): '1',
1871    })
1872    if str(python_bin_path) not in path_str:
1873        pip_os_env['PATH'] = str(python_bin_path.parent) + path_sep + path_str
1874
1875    return pip_os_env