meerschaum.utils.packages

   1#! /usr/bin/env python
   2# -*- coding: utf-8 -*-
   3# vim:fenc=utf-8
   4
   5"""
   6Functions for managing packages and virtual environments reside here.
   7"""
   8
   9from __future__ import annotations
  10
  11import importlib.util, os, pathlib, re
  12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
  13from meerschaum.utils.threading import Lock, RLock
  14from meerschaum.utils.packages._packages import (
  15    packages,
  16    all_packages,
  17    get_install_names,
  18    _MRSM_PACKAGE_ARCHIVES_PREFIX,
  19)
  20from meerschaum.utils.venv import (
  21    activate_venv,
  22    deactivate_venv,
  23    venv_executable,
  24    venv_exec,
  25    venv_exists,
  26    venv_target_path,
  27    inside_venv,
  28    Venv,
  29    init_venv,
  30)
  31
  32_import_module = importlib.import_module
  33_import_hook_venv = None
  34_locks = {
  35    '_pkg_resources_get_distribution': RLock(),
  36    'import_versions': RLock(),
  37    '_checked_for_updates': RLock(),
  38    '_is_installed_first_check': RLock(),
  39    'emitted_pandas_warning': RLock(),
  40}
  41_checked_for_updates = set()
  42_is_installed_first_check: Dict[str, bool] = {}
  43
  44
  45def get_module_path(
  46    import_name: str,
  47    venv: Optional[str] = 'mrsm',
  48    debug: bool = False,
  49    _try_install_name_on_fail: bool = True,
  50) -> Union[pathlib.Path, None]:
  51    """
  52    Get a module's path without importing.
  53    """
  54    import site
  55    if debug:
  56        from meerschaum.utils.debug import dprint
  57    if not _try_install_name_on_fail:
  58        install_name = _import_to_install_name(import_name, with_version=False)
  59        install_name_lower = install_name.lower().replace('-', '_')
  60        import_name_lower = install_name_lower
  61    else:
  62        import_name_lower = import_name.lower().replace('-', '_')
  63
  64    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
  65    if not vtp.exists():
  66        if debug:
  67            dprint(
  68                (
  69                    "Venv '{venv}' does not exist, cannot import "
  70                    + f"'{import_name}'."
  71                ),
  72                color = False,
  73            )
  74        return None
  75
  76    venv_target_candidate_paths = [vtp]
  77    if venv is None:
  78        site_user_packages_dirs = [
  79            pathlib.Path(site.getusersitepackages())
  80        ] if not inside_venv() else []
  81        site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
  82
  83        paths_to_add = [
  84            path
  85            for path in site_user_packages_dirs + site_packages_dirs
  86            if path not in venv_target_candidate_paths
  87        ]
  88        venv_target_candidate_paths += paths_to_add
  89
  90    candidates = []
  91    for venv_target_candidate in venv_target_candidate_paths:
  92        try:
  93            file_names = os.listdir(venv_target_candidate)
  94        except FileNotFoundError:
  95            continue
  96        for file_name in file_names:
  97            file_name_lower = file_name.lower().replace('-', '_')
  98            if not file_name_lower.startswith(import_name_lower):
  99                continue
 100            if file_name.endswith('dist_info'):
 101                continue
 102            file_path = venv_target_candidate / file_name
 103
 104            ### Most likely: Is a directory with __init__.py
 105            if file_name_lower == import_name_lower and file_path.is_dir():
 106                init_path = file_path / '__init__.py'
 107                if init_path.exists():
 108                    candidates.append(init_path)
 109
 110            ### May be a standalone .py file.
 111            elif file_name_lower == import_name_lower + '.py':
 112                candidates.append(file_path)
 113
 114            ### Compiled wheels (e.g. pyodbc)
 115            elif file_name_lower.startswith(import_name_lower + '.'):
 116                candidates.append(file_path)
 117
 118    if len(candidates) == 1:
 119        return candidates[0]
 120
 121    if not candidates:
 122        if _try_install_name_on_fail:
 123            return get_module_path(
 124                import_name, venv=venv, debug=debug,
 125                _try_install_name_on_fail=False
 126            )
 127        return None
 128
 129    specs_paths = []
 130    for candidate_path in candidates:
 131        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
 132        if spec is not None:
 133            return candidate_path
 134    
 135    return None
 136
 137
 138def manually_import_module(
 139    import_name: str,
 140    venv: Optional[str] = 'mrsm',
 141    check_update: bool = True,
 142    check_pypi: bool = False,
 143    install: bool = True,
 144    split: bool = True,
 145    warn: bool = True,
 146    color: bool = True,
 147    debug: bool = False,
 148    use_sys_modules: bool = True,
 149) -> Union['ModuleType', None]:
 150    """
 151    Manually import a module from a virtual environment (or the base environment).
 152
 153    Parameters
 154    ----------
 155    import_name: str
 156        The name of the module.
 157        
 158    venv: Optional[str], default 'mrsm'
 159        The virtual environment to read from.
 160
 161    check_update: bool, default True
 162        If `True`, examine whether the available version of the package meets the required version.
 163
 164    check_pypi: bool, default False
 165        If `True`, check PyPI for updates before importing.
 166
 167    install: bool, default True
 168        If `True`, install the package if it's not installed or needs an update.
 169
 170    split: bool, default True
 171        If `True`, split `import_name` on periods to get the package name.
 172
 173    warn: bool, default True
 174        If `True`, raise a warning if the package cannot be imported.
 175
 176    color: bool, default True
 177        If `True`, use color output for debug and warning text.
 178
 179    debug: bool, default False
 180        Verbosity toggle.
 181
 182    use_sys_modules: bool, default True
 183        If `True`, return the module in `sys.modules` if it exists.
 184        Otherwise continue with manually importing.
 185
 186    Returns
 187    -------
 188    The specified module or `None` if it can't be imported.
 189
 190    """
 191    import sys
 192    _previously_imported = import_name in sys.modules
 193    if _previously_imported and use_sys_modules:
 194        return sys.modules[import_name]
 195
 196    from meerschaum.utils.warnings import warn as warn_function
 197    import warnings
 198    root_name = import_name.split('.')[0] if split else import_name
 199    install_name = _import_to_install_name(root_name)
 200
 201    root_path = get_module_path(root_name, venv=venv)
 202    if root_path is None:
 203        return None
 204
 205    mod_path = root_path
 206    if mod_path.is_dir():
 207        for _dir in import_name.split('.')[:-1]:
 208            mod_path = mod_path / _dir
 209            possible_end_module_filename = import_name.split('.')[-1] + '.py'
 210            try:
 211                mod_path = (
 212                    (mod_path / possible_end_module_filename)
 213                    if possible_end_module_filename in os.listdir(mod_path)
 214                    else (
 215                        mod_path / import_name.split('.')[-1] / '__init__.py'
 216                    )
 217                )
 218            except Exception:
 219                mod_path = None
 220
 221    spec = (
 222        importlib.util.find_spec(import_name)
 223        if mod_path is None or not mod_path.exists()
 224        else importlib.util.spec_from_file_location(import_name, str(mod_path))
 225    )
 226    root_spec = (
 227        importlib.util.find_spec(root_name)
 228        if not root_path.exists()
 229        else importlib.util.spec_from_file_location(root_name, str(root_path))
 230    )
 231
 232    ### Check for updates before importing.
 233    _version = (
 234        determine_version(
 235            pathlib.Path(root_spec.origin),
 236            import_name=root_name, venv=venv, debug=debug
 237        ) if root_spec is not None and root_spec.origin is not None else None
 238    )
 239
 240    if _version is not None:
 241        if check_update:
 242            if need_update(
 243                None,
 244                import_name=root_name,
 245                version=_version,
 246                check_pypi=check_pypi,
 247                debug=debug,
 248            ):
 249                if install:
 250                    if not pip_install(
 251                        root_name,
 252                        venv=venv,
 253                        split=False,
 254                        check_update=check_update,
 255                        color=color,
 256                        debug=debug
 257                    ) and warn:
 258                        warn_function(
 259                            f"There's an update available for '{install_name}', "
 260                            + "but it failed to install. "
 261                            + "Try installig via Meerschaum with "
 262                            + "`install packages '{install_name}'`.",
 263                            ImportWarning,
 264                            stacklevel=3,
 265                            color=False,
 266                        )
 267                elif warn:
 268                    warn_function(
 269                        f"There's an update available for '{root_name}'.",
 270                        stack=False,
 271                        color=False,
 272                    )
 273                spec = (
 274                    importlib.util.find_spec(import_name)
 275                    if mod_path is None or not mod_path.exists()
 276                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
 277                )
 278
 279    if spec is None:
 280        try:
 281            mod = _import_module(import_name)
 282        except Exception:
 283            mod = None
 284        return mod
 285
 286    with Venv(venv, debug=debug):
 287        mod = importlib.util.module_from_spec(spec)
 288        old_sys_mod = sys.modules.get(import_name, None)
 289        sys.modules[import_name] = mod
 290
 291        try:
 292            with warnings.catch_warnings():
 293                warnings.filterwarnings('ignore', 'The NumPy')
 294                spec.loader.exec_module(mod)
 295        except Exception:
 296            pass
 297        mod = _import_module(import_name)
 298        if old_sys_mod is not None:
 299            sys.modules[import_name] = old_sys_mod
 300        else:
 301            del sys.modules[import_name]
 302
 303    return mod
 304
 305
 306def _import_to_install_name(import_name: str, with_version: bool = True) -> str:
 307    """
 308    Try to translate an import name to an installation name.
 309    """
 310    install_name = all_packages.get(import_name, import_name)
 311    if with_version:
 312        return install_name
 313    return get_install_no_version(install_name)
 314
 315
 316def _import_to_dir_name(import_name: str) -> str:
 317    """
 318    Translate an import name to the package name in the sites-packages directory.
 319    """
 320    import re
 321    return re.split(
 322        r'[<>=\[]', all_packages.get(import_name, import_name)
 323    )[0].replace('-', '_').lower() 
 324
 325
 326def _install_to_import_name(install_name: str) -> str:
 327    """
 328    Translate an installation name to a package's import name.
 329    """
 330    _install_no_version = get_install_no_version(install_name)
 331    return get_install_names().get(_install_no_version, _install_no_version)
 332
 333
 334def get_install_no_version(install_name: str) -> str:
 335    """
 336    Strip the version information from the install name.
 337    """
 338    import re
 339    return re.split(r'[\[=<>,! \]]', install_name)[0]
 340
 341
 342import_versions = {}
 343def determine_version(
 344    path: pathlib.Path,
 345    import_name: Optional[str] = None,
 346    venv: Optional[str] = 'mrsm',
 347    search_for_metadata: bool = True,
 348    split: bool = True,
 349    warn: bool = False,
 350    debug: bool = False,
 351) -> Union[str, None]:
 352    """
 353    Determine a module's `__version__` string from its filepath.
 354    
 355    First it searches for pip metadata, then it attempts to import the module in a subprocess.
 356
 357    Parameters
 358    ----------
 359    path: pathlib.Path
 360        The file path of the module.
 361
 362    import_name: Optional[str], default None
 363        The name of the module. If omitted, it will be determined from the file path.
 364        Defaults to `None`.
 365
 366    venv: Optional[str], default 'mrsm'
 367        The virtual environment of the Python interpreter to use if importing is necessary.
 368
 369    search_for_metadata: bool, default True
 370        If `True`, search the pip site_packages directory (assumed to be the parent)
 371        for the corresponding dist-info directory.
 372
 373    warn: bool, default True
 374        If `True`, raise a warning if the module fails to import in the subprocess.
 375
 376    split: bool, default True
 377        If `True`, split the determined import name by periods to get the room name.
 378
 379    Returns
 380    -------
 381    The package's version string if available or `None`.
 382    If multiple versions are found, it will trigger an import in a subprocess.
 383
 384    """
 385    with _locks['import_versions']:
 386        if venv not in import_versions:
 387            import_versions[venv] = {}
 388    import os
 389    old_cwd = os.getcwd()
 390    from meerschaum.utils.warnings import warn as warn_function
 391    if import_name is None:
 392        import_name = path.parent.stem if path.stem == '__init__' else path.stem
 393        import_name = import_name.split('.')[0] if split else import_name
 394    if import_name in import_versions[venv]:
 395        return import_versions[venv][import_name]
 396    _version = None
 397    module_parent_dir = (
 398        path.parent.parent if path.stem == '__init__' else path.parent
 399    ) if path is not None else venv_target_path(venv, allow_nonexistent=True, debug=debug)
 400
 401    if not module_parent_dir.exists():
 402        return None
 403
 404    installed_dir_name = _import_to_dir_name(import_name)
 405    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
 406
 407    ### First, check if a dist-info directory exists.
 408    _found_versions = []
 409    if search_for_metadata:
 410        try:
 411            filenames = os.listdir(module_parent_dir)
 412        except FileNotFoundError:
 413            filenames = []
 414        for filename in filenames:
 415            if not filename.endswith('.dist-info'):
 416                continue
 417            filename_lower = filename.lower()
 418            if not filename_lower.startswith(clean_installed_dir_name + '-'):
 419                continue
 420            _v = filename.replace('.dist-info', '').split("-")[-1]
 421            _found_versions.append(_v)
 422
 423    if len(_found_versions) == 1:
 424        _version = _found_versions[0]
 425        with _locks['import_versions']:
 426            import_versions[venv][import_name] = _version
 427        return _found_versions[0]
 428
 429    if not _found_versions:
 430        try:
 431            import importlib.metadata as importlib_metadata
 432        except ImportError:
 433            importlib_metadata = attempt_import(
 434                'importlib_metadata',
 435                debug=debug, check_update=False, precheck=False,
 436                color=False, check_is_installed=False, lazy=False,
 437            )
 438        try:
 439            os.chdir(module_parent_dir)
 440            _version = importlib_metadata.metadata(import_name)['Version']
 441        except Exception:
 442            _version = None
 443        finally:
 444            os.chdir(old_cwd)
 445
 446        if _version is not None:
 447            with _locks['import_versions']:
 448                import_versions[venv][import_name] = _version
 449            return _version
 450
 451    if debug:
 452        print(f'Found multiple versions for {import_name}: {_found_versions}')
 453
 454    module_parent_dir_str = module_parent_dir.as_posix()
 455
 456    ### Not a pip package, so let's try importing the module directly (in a subprocess).
 457    _no_version_str = 'no-version'
 458    code = (
 459        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
 460        + f"module = importlib.import_module('{import_name}');\n"
 461        + "try:\n"
 462        + "  print(module.__version__ , end='')\n"
 463        + "except:\n"
 464        + f"  print('{_no_version_str}', end='')"
 465    )
 466    exit_code, stdout_bytes, stderr_bytes = venv_exec(
 467        code, venv=venv, with_extras=True, debug=debug
 468    )
 469    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
 470    _version = stdout.split('\n')[-1] if exit_code == 0 else None
 471    _version = _version if _version != _no_version_str else None
 472
 473    if _version is None:
 474        _version = _get_package_metadata(import_name, venv).get('version', None)
 475    if _version is None and warn:
 476        warn_function(
 477            f"Failed to determine a version for '{import_name}':\n{stderr}",
 478            stack = False
 479        )
 480
 481    ### If `__version__` doesn't exist, return `None`.
 482    import_versions[venv][import_name] = _version
 483    return _version
 484
 485
 486def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]:
 487    """
 488    Get a package's metadata from pip.
 489    This is useful for getting a version when no `__version__` is defined
 490    and multiple versions are installed.
 491
 492    Parameters
 493    ----------
 494    import_name: str
 495        The package's import or installation name.
 496
 497    venv: Optional[str]
 498        The virtual environment which contains the package.
 499
 500    Returns
 501    -------
 502    A dictionary of metadata from pip.
 503    """
 504    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 505    install_name = _import_to_install_name(import_name)
 506    if install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX):
 507        return {}
 508    _args = ['pip', 'show', install_name]
 509    if venv is not None:
 510        cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 511        _args += ['--cache-dir', cache_dir_path.as_posix()]
 512
 513    if use_uv():
 514        package_name = 'uv'
 515        _args = ['pip', 'show', install_name]
 516    else:
 517        package_name = 'pip'
 518        _args = ['show', install_name]
 519
 520    proc = run_python_package(
 521        package_name, _args,
 522        capture_output=True, as_proc=True, venv=venv, universal_newlines=True,
 523    )
 524    outs, errs = proc.communicate()
 525    lines = outs.split('\n')
 526    meta = {}
 527    for line in lines:
 528        vals = line.split(": ")
 529        if len(vals) != 2:
 530            continue
 531        k, v = vals[0].lower(), vals[1]
 532        if v and 'UNKNOWN' not in v:
 533            meta[k] = v
 534    return meta
 535
 536
 537def need_update(
 538    package: Optional['ModuleType'] = None,
 539    install_name: Optional[str] = None,
 540    import_name: Optional[str] = None,
 541    version: Optional[str] = None,
 542    check_pypi: bool = False,
 543    split: bool = True,
 544    color: bool = True,
 545    debug: bool = False,
 546    _run_determine_version: bool = True,
 547) -> bool:
 548    """
 549    Check if a Meerschaum dependency needs an update.
 550    Returns a bool for whether or not a package needs to be updated.
 551
 552    Parameters
 553    ----------
 554    package: 'ModuleType'
 555        The module of the package to be updated.
 556
 557    install_name: Optional[str], default None
 558        If provided, use this string to determine the required version.
 559        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
 560
 561    import_name:
 562        If provided, override the package's `__name__` string.
 563
 564    version: Optional[str], default None
 565        If specified, override the package's `__version__` string.
 566
 567    check_pypi: bool, default False
 568        If `True`, check pypi.org for updates.
 569        Defaults to `False`.
 570
 571    split: bool, default True
 572        If `True`, split the module's name on periods to detrive the root name.
 573        Defaults to `True`.
 574
 575    color: bool, default True
 576        If `True`, format debug output.
 577        Defaults to `True`.
 578
 579    debug: bool, default True
 580        Verbosity toggle.
 581
 582    Returns
 583    -------
 584    A bool indicating whether the package requires an update.
 585
 586    """
 587    if debug:
 588        from meerschaum.utils.debug import dprint
 589    from meerschaum.utils.warnings import warn as warn_function
 590    import re
 591    root_name = (
 592        package.__name__.split('.')[0] if split else package.__name__
 593    ) if import_name is None else (
 594        import_name.split('.')[0] if split else import_name
 595    )
 596    install_name = install_name or _import_to_install_name(root_name)
 597    with _locks['_checked_for_updates']:
 598        if install_name in _checked_for_updates:
 599            return False
 600        _checked_for_updates.add(install_name)
 601
 602    _install_no_version = get_install_no_version(install_name)
 603    required_version = (
 604        install_name
 605        .replace(_install_no_version, '')
 606    )
 607    if ']' in required_version:
 608        required_version = required_version.split(']')[1]
 609
 610    ### No minimum version was specified, and we're not going to check PyPI.
 611    if not required_version and not check_pypi:
 612        return False
 613
 614    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
 615    if '.dev' in required_version:
 616        required_version = required_version.split('.dev')[0]
 617    if version and '.dev' in version:
 618        version = version.split('.dev')[0]
 619
 620    try:
 621        if not version:
 622            if not _run_determine_version:
 623                version = determine_version(
 624                    pathlib.Path(package.__file__),
 625                    import_name=root_name, warn=False, debug=debug
 626                )
 627        if version is None:
 628            return False
 629    except Exception as e:
 630        if debug:
 631            dprint(str(e), color=color)
 632            dprint("No version could be determined from the installed package.", color=color)
 633        return False
 634    split_version = version.split('.')
 635    last_part = split_version[-1]
 636    if len(split_version) == 2:
 637        version = '.'.join(split_version) + '.0'
 638    elif 'dev' in last_part or 'rc' in last_part:
 639        tag = 'dev' if 'dev' in last_part else 'rc'
 640        last_sep = '-'
 641        if not last_part.startswith(tag):
 642            last_part = f'-{tag}'.join(last_part.split(tag))
 643            last_sep = '.'
 644        version = '.'.join(split_version[:-1]) + last_sep + last_part
 645    elif len(split_version) > 3:
 646        version = '.'.join(split_version[:3])
 647
 648    packaging_version = attempt_import(
 649        'packaging.version', check_update=False, lazy=False, debug=debug,
 650    )
 651
 652    ### Get semver if necessary
 653    if required_version:
 654        semver_path = get_module_path('semver', debug=debug)
 655        if semver_path is None:
 656            no_venv_semver_path = get_module_path('semver', venv=None, debug=debug)
 657            if no_venv_semver_path is None:
 658                pip_install(_import_to_install_name('semver'), debug=debug)
 659        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
 660    if check_pypi:
 661        ### Check PyPI for updates
 662        update_checker = attempt_import(
 663            'update_checker', lazy=False, check_update=False, debug=debug
 664        )
 665        checker = update_checker.UpdateChecker()
 666        result = checker.check(_install_no_version, version)
 667    else:
 668        ### Skip PyPI and assume we can't be sure.
 669        result = None
 670
 671    ### Compare PyPI's version with our own.
 672    if result is not None:
 673        ### We have a result from PyPI and a stated required version.
 674        if required_version:
 675            try:
 676                return semver.Version.parse(result.available_version).match(required_version)
 677            except AttributeError as e:
 678                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 679                semver = manually_import_module('semver', venv='mrsm')
 680                return semver.Version.parse(version).match(required_version)
 681            except Exception as e:
 682                if debug:
 683                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
 684                return False
 685
 686        ### If `check_pypi` and we don't have a required version, check if PyPI's version
 687        ### is newer than the installed version.
 688        else:
 689            return (
 690                packaging_version.parse(result.available_version) > 
 691                packaging_version.parse(version)
 692            )
 693
 694    ### We might be depending on a prerelease.
 695    ### Sanity check that the required version is not greater than the installed version. 
 696    required_version = (
 697        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
 698        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
 699        .replace('-py3-none-any.whl', '')
 700    )
 701
 702    if 'a' in required_version:
 703        required_version = required_version.replace('a', '-pre.').replace('+mrsm', '')
 704        version = version.replace('a', '-pre.').replace('+mrsm', '')
 705    try:
 706        return (
 707            (not semver.Version.parse(version).match(required_version))
 708            if required_version else False
 709        )
 710    except AttributeError:
 711        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 712        semver = manually_import_module('semver', venv='mrsm', debug=debug)
 713        return (
 714            (not semver.Version.parse(version).match(required_version))
 715            if required_version else False
 716        )
 717    except Exception as e:
 718        print(f"Unable to parse version ({version}) for package '{import_name}'.")
 719        print(e)
 720        if debug:
 721            dprint(e)
 722        return False
 723    try:
 724        return (
 725            packaging_version.parse(version) > 
 726            packaging_version.parse(required_version)
 727        )
 728    except Exception as e:
 729        if debug:
 730            dprint(e)
 731        return False
 732    return False
 733
 734
 735def get_pip(
 736    venv: Optional[str] = 'mrsm',
 737    color: bool = True,
 738    debug: bool = False,
 739) -> bool:
 740    """
 741    Download and run the get-pip.py script.
 742
 743    Parameters
 744    ----------
 745    venv: Optional[str], default 'mrsm'
 746        The virtual environment into which to install `pip`.
 747
 748    color: bool, default True
 749        If `True`, force color output.
 750
 751    debug: bool, default False
 752        Verbosity toggle.
 753
 754    Returns
 755    -------
 756    A bool indicating success.
 757
 758    """
 759    import sys
 760    import subprocess
 761    from meerschaum.utils.misc import wget
 762    from meerschaum.config._paths import CACHE_RESOURCES_PATH
 763    from meerschaum._internal.static import STATIC_CONFIG
 764    url = STATIC_CONFIG['system']['urls']['get-pip.py']
 765    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
 766    try:
 767        wget(url, dest, color=False, debug=debug)
 768    except Exception:
 769        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
 770        sys.exit(1)
 771    if venv is not None:
 772        init_venv(venv=venv, debug=debug)
 773    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
 774    return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0
 775
 776
 777def pip_install(
 778    *install_names: str,
 779    args: Optional[List[str]] = None,
 780    requirements_file_path: Union[pathlib.Path, str, None] = None,
 781    venv: Optional[str] = 'mrsm',
 782    split: bool = False,
 783    check_update: bool = True,
 784    check_pypi: bool = True,
 785    check_wheel: bool = True,
 786    _uninstall: bool = False,
 787    _from_completely_uninstall: bool = False,
 788    _install_uv_pip: bool = True,
 789    _use_uv_pip: bool = True,
 790    color: bool = True,
 791    silent: bool = False,
 792    debug: bool = False,
 793) -> bool:
 794    """
 795    Install packages from PyPI with `pip`.
 796
 797    Parameters
 798    ----------
 799    *install_names: str
 800        The installation names of packages to be installed.
 801        This includes version restrictions.
 802        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 803        from its import name.
 804
 805    args: Optional[List[str]], default None
 806        A list of command line arguments to pass to `pip`.
 807        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 808
 809    requirements_file_path: Optional[pathlib.Path, str], default None
 810        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 811
 812    venv: str, default 'mrsm'
 813        The virtual environment to install into.
 814
 815    split: bool, default False
 816        If `True`, split on periods and only install the root package name.
 817
 818    check_update: bool, default True
 819        If `True`, check if the package requires an update.
 820
 821    check_pypi: bool, default True
 822        If `True` and `check_update` is `True`, check PyPI for the latest version.
 823
 824    check_wheel: bool, default True
 825        If `True`, check if `wheel` is available.
 826
 827    _uninstall: bool, default False
 828        If `True`, uninstall packages instead.
 829
 830    color: bool, default True
 831        If `True`, include color in debug text.
 832
 833    silent: bool, default False
 834        If `True`, skip printing messages.
 835
 836    debug: bool, default False
 837        Verbosity toggle.
 838
 839    Returns
 840    -------
 841    A bool indicating success.
 842
 843    """
 844    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 845    from meerschaum._internal.static import STATIC_CONFIG
 846    from meerschaum.utils.warnings import warn
 847    if args is None:
 848        args = ['--upgrade'] if not _uninstall else []
 849    ANSI = True if color else False
 850    if check_wheel:
 851        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 852
 853    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
 854    inside_daemon = daemon_env_var in os.environ
 855    if inside_daemon:
 856        silent = True
 857
 858    _args = list(args)
 859    have_pip = venv_contains_package('pip', venv=None, debug=debug)
 860    pip_venv = None
 861    try:
 862        import pip
 863        have_pip = True
 864    except ImportError:
 865        have_pip = False
 866    try:
 867        import uv
 868        uv_bin = uv.find_uv_bin()
 869        have_uv_pip = True
 870    except (ImportError, FileNotFoundError):
 871        uv_bin = None
 872        have_uv_pip = False
 873
 874    if have_pip and not have_uv_pip and _install_uv_pip and is_uv_enabled():
 875        if not pip_install(
 876            'uv', 'PyYAML',
 877            venv=None,
 878            debug=debug,
 879            _install_uv_pip=False,
 880            check_update=False,
 881            check_pypi=False,
 882            check_wheel=False,
 883        ) and not silent:
 884            warn(
 885                f"Failed to install `uv` for virtual environment '{venv}'.",
 886                color=False,
 887            )
 888
 889    use_uv_pip = (
 890        _use_uv_pip
 891        and venv_contains_package('uv', venv=None, debug=debug)
 892        and uv_bin is not None
 893        and venv is not None
 894        and is_uv_enabled()
 895    )
 896
 897    import sys
 898    if not have_pip and not use_uv_pip:
 899        have_mrsm_pip = venv_contains_package('pip', venv='mrsm')
 900        if not have_mrsm_pip and not get_pip(venv=venv, color=color, debug=debug):
 901            import sys
 902            minor = sys.version_info.minor
 903            print(
 904                "\nFailed to import `pip` and `ensurepip`.\n"
 905                + "If you are running Ubuntu/Debian, "
 906                + f"you might need to install `python3.{minor}-distutils`:\n\n"
 907                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
 908                + "Please install pip and restart Meerschaum.\n\n"
 909                + "You can find instructions on installing `pip` here:\n"
 910                + "https://pip.pypa.io/en/stable/installing/"
 911            )
 912            sys.exit(1)
 913
 914        pip = attempt_import('pip', lazy=False)
 915        pip_venv = 'mrsm'
 916
 917    with Venv(venv, debug=debug):
 918        if venv is not None:
 919            if (
 920                '--ignore-installed' not in args
 921                and '-I' not in _args
 922                and not _uninstall
 923                and not use_uv_pip
 924            ):
 925                _args += ['--ignore-installed']
 926            if '--cache-dir' not in args and not _uninstall:
 927                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 928                _args += ['--cache-dir', str(cache_dir_path)]
 929
 930        if 'pip' not in ' '.join(_args) and not use_uv_pip:
 931            if check_update and not _uninstall:
 932                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
 933                if need_update(pip, check_pypi=check_pypi, debug=debug):
 934                    _args.append(all_packages['pip'])
 935
 936        _args = (['install'] if not _uninstall else ['uninstall']) + _args
 937
 938        if check_wheel and not _uninstall and not use_uv_pip:
 939            if not have_wheel:
 940                setup_packages_to_install = (
 941                    ['setuptools', 'wheel', 'PyYAML']
 942                    + (['uv'] if is_uv_enabled() else [])
 943                )
 944                if not pip_install(
 945                    *setup_packages_to_install,
 946                    venv=venv,
 947                    check_update=False,
 948                    check_pypi=False,
 949                    check_wheel=False,
 950                    debug=debug,
 951                    _install_uv_pip=False,
 952                ) and not silent:
 953                    from meerschaum.utils.misc import items_str
 954                    warn(
 955                        (
 956                            f"Failed to install {items_str(setup_packages_to_install)} for virtual "
 957                            + f"environment '{venv}'."
 958                        ),
 959                        color=False,
 960                    )
 961
 962        if requirements_file_path is not None:
 963            _args.append('-r')
 964            _args.append(pathlib.Path(requirements_file_path).resolve().as_posix())
 965
 966        if not ANSI and '--no-color' not in _args:
 967            _args.append('--no-color')
 968
 969        if '--no-input' not in _args and not use_uv_pip:
 970            _args.append('--no-input')
 971
 972        if _uninstall and '-y' not in _args and not use_uv_pip:
 973            _args.append('-y')
 974
 975        if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip:
 976            _args.append('--no-warn-conflicts')
 977
 978        if '--disable-pip-version-check' not in _args and not use_uv_pip:
 979            _args.append('--disable-pip-version-check')
 980
 981        if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
 982            if venv is not None:
 983                vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 984                if not vtp.exists():
 985                    if not init_venv(venv, force=True):
 986                        vtp.mkdir(parents=True, exist_ok=True)
 987                _args += ['--target', venv_target_path(venv, debug=debug).as_posix()]
 988        elif (
 989            '--target' not in _args
 990                and '-t' not in _args
 991                and not inside_venv()
 992                and not _uninstall
 993                and not use_uv_pip
 994        ):
 995            _args.append('--user')
 996
 997        if venv is None and '--break-system-packages' not in _args:
 998            _args.append('--break-system-packages')
 999
1000        if debug:
1001            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
1002                if use_uv_pip:
1003                    _args.append('--verbose')
1004        else:
1005            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
1006                pass
1007
1008        _packages = [
1009            (
1010                get_install_no_version(install_name)
1011                if _uninstall or install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX)
1012                else install_name
1013            )
1014            for install_name in install_names
1015        ]
1016        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
1017        for p in _packages:
1018            msg += f'\n  - {p}'
1019        if not silent:
1020            print(msg)
1021
1022        if _uninstall and not _from_completely_uninstall and not use_uv_pip:
1023            for install_name in _packages:
1024                _install_no_version = get_install_no_version(install_name)
1025                if _install_no_version in ('pip', 'wheel', 'uv'):
1026                    continue
1027                if not completely_uninstall_package(
1028                    _install_no_version,
1029                    venv=venv,
1030                    debug=debug,
1031                ) and not silent:
1032                    warn(
1033                        f"Failed to clean up package '{_install_no_version}'.",
1034                    )
1035
1036        ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease.
1037        if use_uv_pip:
1038            _args.insert(0, 'pip')
1039            if not _uninstall and get_prerelease_dependencies(_packages):
1040                _args.append('--prerelease=allow')
1041
1042        rc = run_python_package(
1043            ('pip' if not use_uv_pip else 'uv'),
1044            _args + _packages,
1045            venv=pip_venv,
1046            env=_get_pip_os_env(color=color),
1047            debug=debug,
1048        )
1049        if debug:
1050            print(f"{rc=}")
1051        success = rc == 0
1052
1053    msg = (
1054        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
1055        else "Failed to " + ('un' if _uninstall else '') + "install packages."
1056    )
1057    if not silent:
1058        print(msg)
1059    if debug and not silent:
1060        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
1061    return success
1062
1063
1064def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1065    """
1066    Return a list of explicitly prerelease dependencies from a list of packages.
1067    """
1068    if _packages is None:
1069        _packages = list(all_packages.keys())
1070    prelrease_strings = ['dev', 'rc', 'a']
1071    prerelease_packages = []
1072    for install_name in _packages:
1073        _install_no_version = get_install_no_version(install_name)
1074        import_name = _install_to_import_name(install_name)
1075        install_with_version = _import_to_install_name(import_name)
1076        version_only = (
1077            install_with_version.lower().replace(_install_no_version.lower(), '')
1078            .split(']')[-1]
1079        )
1080
1081        is_prerelease = False
1082        for prelrease_string in prelrease_strings:
1083            if prelrease_string in version_only:
1084                is_prerelease = True
1085
1086        if is_prerelease:
1087            prerelease_packages.append(install_name)
1088    return prerelease_packages
1089
1090
1091def completely_uninstall_package(
1092    install_name: str,
1093    venv: str = 'mrsm',
1094    debug: bool = False,
1095) -> bool:
1096    """
1097    Continue calling `pip uninstall` until a package is completely
1098    removed from a virtual environment. 
1099    This is useful for dealing with multiple installed versions of a package.
1100    """
1101    attempts = 0
1102    _install_no_version = get_install_no_version(install_name)
1103    clean_install_no_version = _install_no_version.lower().replace('-', '_')
1104    installed_versions = []
1105    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1106    if not vtp.exists():
1107        return True
1108
1109    for file_name in os.listdir(vtp):
1110        if not file_name.endswith('.dist-info'):
1111            continue
1112        clean_dist_info = file_name.replace('-', '_').lower()
1113        if not clean_dist_info.startswith(clean_install_no_version):
1114            continue
1115        installed_versions.append(file_name)
1116
1117    max_attempts = len(installed_versions)
1118    while attempts < max_attempts:
1119        if not venv_contains_package(
1120            _install_to_import_name(_install_no_version),
1121            venv=venv, debug=debug,
1122        ):
1123            return True
1124        if not pip_uninstall(
1125            _install_no_version,
1126            venv = venv,
1127            silent = (not debug),
1128            _from_completely_uninstall = True,
1129            debug = debug,
1130        ):
1131            return False
1132        attempts += 1
1133    return False
1134
1135
1136def pip_uninstall(
1137    *args, **kw
1138) -> bool:
1139    """
1140    Uninstall Python packages.
1141    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
1142    """
1143    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
1144
1145
1146def run_python_package(
1147    package_name: str,
1148    args: Optional[List[str]] = None,
1149    venv: Optional[str] = 'mrsm',
1150    cwd: Optional[str] = None,
1151    env: Optional[Dict[str, str]] = None,
1152    foreground: bool = False,
1153    as_proc: bool = False,
1154    capture_output: bool = False,
1155    debug: bool = False,
1156    **kw: Any,
1157) -> Union[int, subprocess.Popen, None]:
1158    """
1159    Runs an installed python package.
1160    E.g. Translates to `/usr/bin/python -m [package]`
1161
1162    Parameters
1163    ----------
1164    package_name: str
1165        The Python module to be executed.
1166
1167    args: Optional[List[str]], default None
1168        Additional command line arguments to be appended after `-m [package]`.
1169
1170    venv: Optional[str], default 'mrsm'
1171        If specified, execute the Python interpreter from a virtual environment.
1172
1173    cwd: Optional[str], default None
1174        If specified, change directories before starting the process.
1175        Defaults to `None`.
1176
1177    env: Optional[Dict[str, str]], default None
1178        If specified, only use the provided dictionary for the environment variables.
1179        Defaults to `os.environ`.
1180
1181    as_proc: bool, default False
1182        If `True`, return a `subprocess.Popen` object.
1183
1184    capture_output: bool, default False
1185        If `as_proc` is `True`, capture stdout and stderr.
1186
1187    foreground: bool, default False
1188        If `True`, start the subprocess as a foreground process.
1189        Defaults to `False`.
1190
1191    kw: Any
1192        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1193        and by extension `subprocess.Popen()`.
1194
1195    Returns
1196    -------
1197    Either a return code integer or a `subprocess.Popen` object
1198    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1199    """
1200    import sys
1201    import subprocess
1202    import traceback
1203    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1204    from meerschaum.utils.process import run_process
1205    from meerschaum.utils.warnings import warn
1206    if args is None:
1207        args = []
1208    old_cwd = os.getcwd()
1209    if cwd is not None:
1210        os.chdir(cwd)
1211    executable = venv_executable(venv=venv)
1212    venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
1213    env_dict = (env if isinstance(env, dict) else (os.environ or {})).copy()
1214    if venv_path is not None:
1215        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
1216    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1217    if debug:
1218        print(command, file=sys.stderr)
1219    try:
1220        to_return = run_process(
1221            command,
1222            foreground=foreground,
1223            as_proc=as_proc,
1224            capture_output=capture_output,
1225            **kw
1226        )
1227    except Exception:
1228        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1229        warn(msg, color=False)
1230        stdout, stderr = (
1231            (None, None)
1232            if not capture_output
1233            else (subprocess.PIPE, subprocess.PIPE)
1234        )
1235        proc = subprocess.Popen(
1236            command,
1237            stdout=stdout,
1238            stderr=stderr,
1239            stdin=sys.stdin,
1240            env=env_dict,
1241        )
1242        to_return = proc if as_proc else proc.wait()
1243    except KeyboardInterrupt:
1244        to_return = 1 if not as_proc else None
1245    os.chdir(old_cwd)
1246    return to_return
1247
1248
1249def attempt_import(
1250    *names: str,
1251    lazy: bool = True,
1252    warn: bool = True,
1253    install: bool = True,
1254    venv: Optional[str] = 'mrsm',
1255    precheck: bool = True,
1256    split: bool = True,
1257    check_update: bool = False,
1258    check_pypi: bool = False,
1259    check_is_installed: bool = True,
1260    allow_outside_venv: bool = True,
1261    color: bool = True,
1262    debug: bool = False
1263) -> Any:
1264    """
1265    Raise a warning if packages are not installed; otherwise import and return modules.
1266    If `lazy` is `True`, return lazy-imported modules.
1267    
1268    Returns tuple of modules if multiple names are provided, else returns one module.
1269    
1270    Parameters
1271    ----------
1272    names: List[str]
1273        The packages to be imported.
1274
1275    lazy: bool, default True
1276        If `True`, lazily load packages.
1277
1278    warn: bool, default True
1279        If `True`, raise a warning if a package cannot be imported.
1280
1281    install: bool, default True
1282        If `True`, attempt to install a missing package into the designated virtual environment.
1283        If `check_update` is True, install updates if available.
1284
1285    venv: Optional[str], default 'mrsm'
1286        The virtual environment in which to search for packages and to install packages into.
1287
1288    precheck: bool, default True
1289        If `True`, attempt to find module before importing (necessary for checking if modules exist
1290        and retaining lazy imports), otherwise assume lazy is `False`.
1291
1292    split: bool, default True
1293        If `True`, split packages' names on `'.'`.
1294
1295    check_update: bool, default False
1296        If `True` and `install` is `True`, install updates if the required minimum version
1297        does not match.
1298
1299    check_pypi: bool, default False
1300        If `True` and `check_update` is `True`, check PyPI when determining whether
1301        an update is required.
1302
1303    check_is_installed: bool, default True
1304        If `True`, check if the package is contained in the virtual environment.
1305
1306    allow_outside_venv: bool, default True
1307        If `True`, search outside of the specified virtual environment
1308        if the package cannot be found.
1309        Setting to `False` will reinstall the package into a virtual environment, even if it
1310        is installed outside.
1311
1312    color: bool, default True
1313        If `False`, do not print ANSI colors.
1314
1315    Returns
1316    -------
1317    The specified modules. If they're not available and `install` is `True`, it will first
1318    download them into a virtual environment and return the modules.
1319
1320    Examples
1321    --------
1322    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1323    >>> pandas = attempt_import('pandas')
1324
1325    """
1326
1327    import importlib.util
1328
1329    ### to prevent recursion, check if parent Meerschaum package is being imported
1330    if names == ('meerschaum',):
1331        return _import_module('meerschaum')
1332
1333    if venv == 'mrsm' and _import_hook_venv is not None:
1334        if debug:
1335            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1336        venv = _import_hook_venv
1337
1338    _warnings = _import_module('meerschaum.utils.warnings')
1339    warn_function = _warnings.warn
1340
1341    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1342        with Venv(venv=venv, debug=debug):
1343            ### determine the import method (lazy vs normal)
1344            from meerschaum.utils.misc import filter_keywords
1345            import_method = (
1346                _import_module if not lazy
1347                else lazy_import
1348            )
1349            try:
1350                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1351            except Exception as e:
1352                if warn:
1353                    import traceback
1354                    traceback.print_exception(type(e), e, e.__traceback__)
1355                    warn_function(
1356                        f"Failed to import module '{_name}'.\nException:\n{e}",
1357                        ImportWarning,
1358                        stacklevel = (5 if lazy else 4),
1359                        color = False,
1360                    )
1361                mod = None
1362        return mod
1363
1364    modules = []
1365    for name in names:
1366        ### Check if package is a declared dependency.
1367        root_name = name.split('.')[0] if split else name
1368        install_name = _import_to_install_name(root_name)
1369
1370        if install_name is None:
1371            install_name = root_name
1372            if warn and root_name != 'plugins':
1373                warn_function(
1374                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1375                    ImportWarning,
1376                    stacklevel = 3,
1377                    color = False
1378                )
1379
1380        ### Determine if the package exists.
1381        if precheck is False:
1382            found_module = (
1383                do_import(
1384                    name, debug=debug, warn=False, venv=venv, color=color,
1385                    check_update=False, check_pypi=False, split=split,
1386                ) is not None
1387            )
1388        else:
1389            if check_is_installed:
1390                with _locks['_is_installed_first_check']:
1391                    if not _is_installed_first_check.get(name, False):
1392                        package_is_installed = is_installed(
1393                            name,
1394                            venv = venv,
1395                            split = split,
1396                            allow_outside_venv = allow_outside_venv,
1397                            debug = debug,
1398                        )
1399                        _is_installed_first_check[name] = package_is_installed
1400                    else:
1401                        package_is_installed = _is_installed_first_check[name]
1402            else:
1403                package_is_installed = _is_installed_first_check.get(
1404                    name,
1405                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1406                )
1407            found_module = package_is_installed
1408
1409        if not found_module:
1410            if install:
1411                if not pip_install(
1412                    install_name,
1413                    venv = venv,
1414                    split = False,
1415                    check_update = check_update,
1416                    color = color,
1417                    debug = debug
1418                ) and warn:
1419                    warn_function(
1420                        f"Failed to install '{install_name}'.",
1421                        ImportWarning,
1422                        stacklevel = 3,
1423                        color = False,
1424                    )
1425            elif warn:
1426                ### Raise a warning if we can't find the package and install = False.
1427                warn_function(
1428                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1429                     + "some features will not work correctly."
1430                     + "\n\nSet install=True when calling attempt_import.\n"),
1431                    ImportWarning,
1432                    stacklevel = 3,
1433                    color = False,
1434                )
1435
1436        ### Do the import. Will be lazy if lazy=True.
1437        m = do_import(
1438            name, debug=debug, warn=warn, venv=venv, color=color,
1439            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1440        )
1441        modules.append(m)
1442
1443    modules = tuple(modules)
1444    if len(modules) == 1:
1445        return modules[0]
1446    return modules
1447
1448
1449def lazy_import(
1450    name: str,
1451    local_name: str = None,
1452    **kw
1453) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1454    """
1455    Lazily import a package.
1456    """
1457    from meerschaum.utils.packages.lazy_loader import LazyLoader
1458    if local_name is None:
1459        local_name = name
1460    return LazyLoader(
1461        local_name,
1462        globals(),
1463        name,
1464        **kw
1465    )
1466
1467
1468def pandas_name() -> str:
1469    """
1470    Return the configured name for `pandas`.
1471    
1472    Below are the expected possible values:
1473
1474    - 'pandas'
1475    - 'modin.pandas'
1476    - 'dask.dataframe'
1477
1478    """
1479    from meerschaum.config import get_config
1480    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1481    if pandas_module_name == 'modin':
1482        pandas_module_name = 'modin.pandas'
1483    elif pandas_module_name == 'dask':
1484        pandas_module_name = 'dask.dataframe'
1485
1486    return pandas_module_name
1487
1488
1489emitted_pandas_warning: bool = False
1490def import_pandas(
1491    debug: bool = False,
1492    lazy: bool = False,
1493    **kw
1494) -> 'ModuleType':
1495    """
1496    Quality-of-life function to attempt to import the configured version of `pandas`.
1497    """
1498    pandas_module_name = pandas_name()
1499    global emitted_pandas_warning
1500
1501    if pandas_module_name != 'pandas':
1502        with _locks['emitted_pandas_warning']:
1503            if not emitted_pandas_warning:
1504                from meerschaum.utils.warnings import warn
1505                emitted_pandas_warning = True
1506                warn(
1507                    (
1508                        "You are using an alternative Pandas implementation "
1509                        + f"'{pandas_module_name}'"
1510                        + "\n   Features may not work as expected."
1511                    ),
1512                    stack=False,
1513                )
1514
1515    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1516    pandas, pyarrow = attempt_import('pandas', 'pyarrow', debug=debug, lazy=False, **kw)
1517    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1518    return pd
1519
1520
1521def import_rich(
1522    lazy: bool = True,
1523    debug: bool = False,
1524    **kw: Any
1525) -> 'ModuleType':
1526    """
1527    Quality of life function for importing `rich`.
1528    """
1529    from meerschaum.utils.formatting import ANSI, UNICODE
1530    ## need typing_extensions for `from rich import box`
1531    typing_extensions = attempt_import(
1532        'typing_extensions', lazy=False, debug=debug
1533    )
1534    pygments = attempt_import(
1535        'pygments', lazy=False,
1536    )
1537    rich = attempt_import(
1538        'rich', lazy=lazy,
1539        **kw
1540    )
1541    return rich
1542
1543
1544def _dash_less_than_2(**kw) -> bool:
1545    dash = attempt_import('dash', **kw)
1546    if dash is None:
1547        return None
1548    packaging_version = attempt_import('packaging.version', **kw)
1549    return (
1550        packaging_version.parse(dash.__version__) < 
1551        packaging_version.parse('2.0.0')
1552    )
1553
1554
1555def import_dcc(warn=False, **kw) -> 'ModuleType':
1556    """
1557    Import Dash Core Components (`dcc`).
1558    """
1559    return (
1560        attempt_import('dash_core_components', warn=warn, **kw)
1561        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1562    )
1563
1564
1565def import_html(warn=False, **kw) -> 'ModuleType':
1566    """
1567    Import Dash HTML Components (`html`).
1568    """
1569    return (
1570        attempt_import('dash_html_components', warn=warn, **kw)
1571        if _dash_less_than_2(warn=warn, **kw)
1572        else attempt_import('dash.html', warn=warn, **kw)
1573    )
1574
1575
1576def get_modules_from_package(
1577    package: 'package',
1578    names: bool = False,
1579    recursive: bool = False,
1580    lazy: bool = False,
1581    modules_venvs: bool = False,
1582    debug: bool = False
1583):
1584    """
1585    Find and import all modules in a package.
1586    
1587    Returns
1588    -------
1589    Either list of modules or tuple of lists.
1590    """
1591    from os.path import dirname, join, isfile, isdir, basename
1592    import glob
1593
1594    pattern = '*' if recursive else '*.py'
1595    package_path = dirname(package.__file__ or package.__path__[0])
1596    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1597    _all = [
1598        basename(f)[:-3] if isfile(f) else basename(f)
1599        for f in module_names
1600            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1601               and not f.endswith('__init__.py')
1602               and not f.endswith('__pycache__')
1603    ]
1604
1605    if debug:
1606        from meerschaum.utils.debug import dprint
1607        dprint(str(_all))
1608    modules = []
1609    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1610        ### there's probably a better way than a try: catch but it'll do for now
1611        try:
1612            ### if specified, activate the module's virtual environment before importing.
1613            ### NOTE: this only considers the filename, so two modules from different packages
1614            ### may end up sharing virtual environments.
1615            if modules_venvs:
1616                activate_venv(module_name.split('.')[-1], debug=debug)
1617            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1618            modules.append(m)
1619        except Exception as e:
1620            if debug:
1621                dprint(str(e))
1622        finally:
1623            if modules_venvs:
1624                deactivate_venv(module_name.split('.')[-1], debug=debug)
1625    if names:
1626        return _all, modules
1627
1628    return modules
1629
1630
1631def import_children(
1632    package: Optional['ModuleType'] = None,
1633    package_name: Optional[str] = None,
1634    types : Optional[List[str]] = None,
1635    lazy: bool = True,
1636    recursive: bool = False,
1637    debug: bool = False
1638) -> List['ModuleType']:
1639    """
1640    Import all functions in a package to its `__init__`.
1641
1642    Parameters
1643    ----------
1644    package: Optional[ModuleType], default None
1645        Package to import its functions into.
1646        If `None` (default), use parent.
1647
1648    package_name: Optional[str], default None
1649        Name of package to import its functions into
1650        If None (default), use parent.
1651
1652    types: Optional[List[str]], default None
1653        Types of members to return.
1654        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1655
1656    Returns
1657    -------
1658    A list of modules.
1659    """
1660    import sys, inspect
1661
1662    if types is None:
1663        types = ['method', 'builtin', 'function', 'class', 'module']
1664
1665    ### if package_name and package are None, use parent
1666    if package is None and package_name is None:
1667        package_name = inspect.stack()[1][0].f_globals['__name__']
1668
1669    ### populate package or package_name from other other
1670    if package is None:
1671        package = sys.modules[package_name]
1672    elif package_name is None:
1673        package_name = package.__name__
1674
1675    ### Set attributes in sys module version of package.
1676    ### Kinda like setting a dictionary
1677    ###   functions[name] = func
1678    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1679    _all, members = [], []
1680    objects = []
1681    for module in modules:
1682        _objects = []
1683        for ob in inspect.getmembers(module):
1684            for t in types:
1685                ### ob is a tuple of (name, object)
1686                if getattr(inspect, 'is' + t)(ob[1]):
1687                    _objects.append(ob)
1688
1689        if 'module' in types:
1690            _objects.append((module.__name__.split('.')[0], module))
1691        objects += _objects
1692    for ob in objects:
1693        setattr(sys.modules[package_name], ob[0], ob[1])
1694        _all.append(ob[0])
1695        members.append(ob[1])
1696
1697    if debug:
1698        from meerschaum.utils.debug import dprint
1699        dprint(str(_all))
1700    ### set __all__ for import *
1701    setattr(sys.modules[package_name], '__all__', _all)
1702    return members
1703
1704
1705_reload_module_cache = {}
1706def reload_package(
1707    package: str,
1708    skip_submodules: Optional[List[str]] = None,
1709    lazy: bool = False,
1710    debug: bool = False,
1711    **kw: Any
1712):
1713    """
1714    Recursively load a package's subpackages, even if they were not previously loaded.
1715    """
1716    import sys
1717    if isinstance(package, str):
1718        package_name = package
1719    else:
1720        try:
1721            package_name = package.__name__
1722        except Exception as e:
1723            package_name = str(package)
1724
1725    skip_submodules = skip_submodules or []
1726    if 'meerschaum.utils.packages' not in skip_submodules:
1727        skip_submodules.append('meerschaum.utils.packages')
1728    def safeimport():
1729        subs = [
1730            m for m in sys.modules
1731            if m.startswith(package_name + '.')
1732        ]
1733        subs_to_skip = []
1734        for skip_mod in skip_submodules:
1735            for mod in subs:
1736                if mod.startswith(skip_mod):
1737                    subs_to_skip.append(mod)
1738                    continue
1739
1740        subs = [m for m in subs if m not in subs_to_skip]
1741        for module_name in subs:
1742            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1743        if not subs_to_skip:
1744            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1745
1746        return _import_module(package_name)
1747
1748    return safeimport()
1749
1750
1751def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1752    """
1753    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1754    """
1755    reload_package(
1756        'meerschaum',
1757        skip_submodules = [
1758            'meerschaum._internal.shell',
1759            'meerschaum.utils.pool',
1760        ]
1761    )
1762
1763    from meerschaum.plugins import reload_plugins
1764    from meerschaum._internal.shell.Shell import _insert_shell_actions
1765    reload_plugins(debug=debug)
1766    _insert_shell_actions()
1767    return True, "Success"
1768
1769
1770def is_installed(
1771    import_name: str,
1772    venv: Optional[str] = 'mrsm',
1773    split: bool = True,
1774    allow_outside_venv: bool = True,
1775    debug: bool = False,
1776) -> bool:
1777    """
1778    Check whether a package is installed.
1779
1780    Parameters
1781    ----------
1782    import_name: str
1783        The import name of the module.
1784
1785    venv: Optional[str], default 'mrsm'
1786        The venv in which to search for the module.
1787
1788    split: bool, default True
1789        If `True`, split on periods to determine the root module name.
1790
1791    allow_outside_venv: bool, default True
1792        If `True`, search outside of the specified virtual environment
1793        if the package cannot be found.
1794
1795    Returns
1796    -------
1797    A bool indicating whether a package may be imported.
1798    """
1799    if debug:
1800        from meerschaum.utils.debug import dprint
1801    root_name = import_name.split('.')[0] if split else import_name
1802    import importlib.util
1803    with Venv(venv, debug=debug):
1804        try:
1805            spec_path = pathlib.Path(
1806                get_module_path(root_name, venv=venv, debug=debug)
1807                or
1808                (
1809                    importlib.util.find_spec(root_name).origin 
1810                    if venv is not None and allow_outside_venv
1811                    else None
1812                )
1813            )
1814        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1815            spec_path = None
1816
1817        found = (
1818            not need_update(
1819                None,
1820                import_name=root_name,
1821                _run_determine_version=False,
1822                check_pypi=False,
1823                version=determine_version(
1824                    spec_path,
1825                    venv=venv,
1826                    debug=debug,
1827                    import_name=root_name,
1828                ),
1829                debug=debug,
1830            )
1831        ) if spec_path is not None else False
1832
1833    return found
1834
1835
1836def venv_contains_package(
1837    import_name: str,
1838    venv: Optional[str] = 'mrsm',
1839    split: bool = True,
1840    debug: bool = False,
1841) -> bool:
1842    """
1843    Search the contents of a virtual environment for a package.
1844    """
1845    import site
1846    import pathlib
1847    root_name = import_name.split('.')[0] if split else import_name
1848    return get_module_path(root_name, venv=venv, debug=debug) is not None
1849
1850
1851def package_venv(package: 'ModuleType') -> Union[str, None]:
1852    """
1853    Inspect a package and return the virtual environment in which it presides.
1854    """
1855    import os
1856    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1857    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1858        return None
1859    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
1860
1861
1862def ensure_readline() -> 'ModuleType':
1863    """Make sure that the `readline` package is able to be imported."""
1864    import sys
1865    try:
1866        import readline
1867    except ImportError:
1868        readline = None
1869
1870    if readline is None:
1871        import platform
1872        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1873        try:
1874            rl = attempt_import(
1875                rl_name,
1876                lazy=False,
1877                install=True,
1878                venv=None,
1879                warn=False,
1880            )
1881        except (ImportError, ModuleNotFoundError):
1882            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1883                print(f"Unable to import {rl_name}!", file=sys.stderr)
1884                sys.exit(1)
1885
1886    sys.modules['readline'] = readline
1887    return readline
1888
1889
1890def _get_pip_os_env(color: bool = True):
1891    """
1892    Return the environment variables context in which `pip` should be run.
1893    See PEP 668 for why we are overriding the environment.
1894    """
1895    import os, sys, platform
1896    python_bin_path = pathlib.Path(sys.executable)
1897    pip_os_env = os.environ.copy()
1898    path_str = pip_os_env.get('PATH', '') or ''
1899    path_sep = ':' if platform.system() != 'Windows' else ';'
1900    pip_os_env.update({
1901        'PIP_BREAK_SYSTEM_PACKAGES': 'true',
1902        'UV_BREAK_SYSTEM_PACKAGES': 'true',
1903        ('FORCE_COLOR' if color else 'NO_COLOR'): '1',
1904    })
1905    if str(python_bin_path) not in path_str:
1906        pip_os_env['PATH'] = str(python_bin_path.parent) + path_sep + path_str
1907
1908    return pip_os_env
1909
1910
1911def use_uv() -> bool:
1912    """
1913    Return whether `uv` is available and enabled.
1914    """
1915    from meerschaum.utils.misc import is_android
1916    if is_android():
1917        return False
1918
1919    if not is_uv_enabled():
1920        return False
1921
1922    try:
1923        import uv
1924        uv_bin = uv.find_uv_bin()
1925    except (ImportError, FileNotFoundError):
1926        uv_bin = None
1927
1928    if uv_bin is None:
1929        return False
1930
1931    return True
1932
1933
1934def is_uv_enabled() -> bool:
1935    """
1936    Return whether the user has disabled `uv`.
1937    """
1938    from meerschaum.utils.misc import is_android
1939    if is_android():
1940        return False
1941
1942    try:
1943        import yaml
1944    except ImportError:
1945        return False
1946
1947    from meerschaum.config import get_config
1948    enabled = get_config('system', 'experimental', 'uv_pip')
1949    return enabled