meerschaum.utils.packages

Functions for managing packages and virtual environments reside here.

   1#! /usr/bin/env python
   2# -*- coding: utf-8 -*-
   3# vim:fenc=utf-8
   4
   5"""
   6Functions for managing packages and virtual environments reside here.
   7"""
   8
   9from __future__ import annotations
  10
  11import importlib.util, os, pathlib, re
  12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
  13from meerschaum.utils.threading import Lock, RLock
  14from meerschaum.utils.packages._packages import (
  15    packages,
  16    all_packages,
  17    get_install_names,
  18    _MRSM_PACKAGE_ARCHIVES_PREFIX,
  19)
  20from meerschaum.utils.venv import (
  21    activate_venv,
  22    deactivate_venv,
  23    venv_executable,
  24    venv_exec,
  25    venv_exists,
  26    venv_target_path,
  27    inside_venv,
  28    Venv,
  29    init_venv,
  30)
  31
  32_import_module = importlib.import_module
  33_import_hook_venv = None
  34_locks = {
  35    '_pkg_resources_get_distribution': RLock(),
  36    'import_versions': RLock(),
  37    '_checked_for_updates': RLock(),
  38    '_is_installed_first_check': RLock(),
  39    'emitted_pandas_warning': RLock(),
  40}
  41_checked_for_updates = set()
  42_is_installed_first_check: Dict[str, bool] = {}
  43
  44
  45def get_module_path(
  46    import_name: str,
  47    venv: Optional[str] = 'mrsm',
  48    debug: bool = False,
  49    _try_install_name_on_fail: bool = True,
  50) -> Union[pathlib.Path, None]:
  51    """
  52    Get a module's path without importing.
  53    """
  54    import site
  55    if debug:
  56        from meerschaum.utils.debug import dprint
  57    if not _try_install_name_on_fail:
  58        install_name = _import_to_install_name(import_name, with_version=False)
  59        install_name_lower = install_name.lower().replace('-', '_')
  60        import_name_lower = install_name_lower
  61    else:
  62        import_name_lower = import_name.lower().replace('-', '_')
  63
  64    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
  65    if not vtp.exists():
  66        if debug:
  67            dprint(
  68                (
  69                    "Venv '{venv}' does not exist, cannot import "
  70                    + f"'{import_name}'."
  71                ),
  72                color = False,
  73            )
  74        return None
  75
  76    venv_target_candidate_paths = [vtp]
  77    if venv is None:
  78        site_user_packages_dirs = [
  79            pathlib.Path(site.getusersitepackages())
  80        ] if not inside_venv() else []
  81        site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
  82
  83        paths_to_add = [
  84            path
  85            for path in site_user_packages_dirs + site_packages_dirs
  86            if path not in venv_target_candidate_paths
  87        ]
  88        venv_target_candidate_paths += paths_to_add
  89
  90    candidates = []
  91    for venv_target_candidate in venv_target_candidate_paths:
  92        try:
  93            file_names = os.listdir(venv_target_candidate)
  94        except FileNotFoundError:
  95            continue
  96        for file_name in file_names:
  97            file_name_lower = file_name.lower().replace('-', '_')
  98            if not file_name_lower.startswith(import_name_lower):
  99                continue
 100            if file_name.endswith('dist_info'):
 101                continue
 102            file_path = venv_target_candidate / file_name
 103
 104            ### Most likely: Is a directory with __init__.py
 105            if file_name_lower == import_name_lower and file_path.is_dir():
 106                init_path = file_path / '__init__.py'
 107                if init_path.exists():
 108                    candidates.append(init_path)
 109
 110            ### May be a standalone .py file.
 111            elif file_name_lower == import_name_lower + '.py':
 112                candidates.append(file_path)
 113
 114            ### Compiled wheels (e.g. pyodbc)
 115            elif file_name_lower.startswith(import_name_lower + '.'):
 116                candidates.append(file_path)
 117
 118    if len(candidates) == 1:
 119        return candidates[0]
 120
 121    if not candidates:
 122        if _try_install_name_on_fail:
 123            return get_module_path(
 124                import_name, venv=venv, debug=debug,
 125                _try_install_name_on_fail=False
 126            )
 127        return None
 128
 129    specs_paths = []
 130    for candidate_path in candidates:
 131        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
 132        if spec is not None:
 133            return candidate_path
 134    
 135    return None
 136
 137
 138def manually_import_module(
 139    import_name: str,
 140    venv: Optional[str] = 'mrsm',
 141    check_update: bool = True,
 142    check_pypi: bool = False,
 143    install: bool = True,
 144    split: bool = True,
 145    warn: bool = True,
 146    color: bool = True,
 147    debug: bool = False,
 148    use_sys_modules: bool = True,
 149) -> Union['ModuleType', None]:
 150    """
 151    Manually import a module from a virtual environment (or the base environment).
 152
 153    Parameters
 154    ----------
 155    import_name: str
 156        The name of the module.
 157        
 158    venv: Optional[str], default 'mrsm'
 159        The virtual environment to read from.
 160
 161    check_update: bool, default True
 162        If `True`, examine whether the available version of the package meets the required version.
 163
 164    check_pypi: bool, default False
 165        If `True`, check PyPI for updates before importing.
 166
 167    install: bool, default True
 168        If `True`, install the package if it's not installed or needs an update.
 169
 170    split: bool, default True
 171        If `True`, split `import_name` on periods to get the package name.
 172
 173    warn: bool, default True
 174        If `True`, raise a warning if the package cannot be imported.
 175
 176    color: bool, default True
 177        If `True`, use color output for debug and warning text.
 178
 179    debug: bool, default False
 180        Verbosity toggle.
 181
 182    use_sys_modules: bool, default True
 183        If `True`, return the module in `sys.modules` if it exists.
 184        Otherwise continue with manually importing.
 185
 186    Returns
 187    -------
 188    The specified module or `None` if it can't be imported.
 189
 190    """
 191    import sys
 192    _previously_imported = import_name in sys.modules
 193    if _previously_imported and use_sys_modules:
 194        return sys.modules[import_name]
 195
 196    from meerschaum.utils.warnings import warn as warn_function
 197    import warnings
 198    root_name = import_name.split('.')[0] if split else import_name
 199    install_name = _import_to_install_name(root_name)
 200
 201    root_path = get_module_path(root_name, venv=venv)
 202    if root_path is None:
 203        return None
 204
 205    mod_path = root_path
 206    if mod_path.is_dir():
 207        for _dir in import_name.split('.')[:-1]:
 208            mod_path = mod_path / _dir
 209            possible_end_module_filename = import_name.split('.')[-1] + '.py'
 210            try:
 211                mod_path = (
 212                    (mod_path / possible_end_module_filename)
 213                    if possible_end_module_filename in os.listdir(mod_path)
 214                    else (
 215                        mod_path / import_name.split('.')[-1] / '__init__.py'
 216                    )
 217                )
 218            except Exception:
 219                mod_path = None
 220
 221    spec = (
 222        importlib.util.find_spec(import_name)
 223        if mod_path is None or not mod_path.exists()
 224        else importlib.util.spec_from_file_location(import_name, str(mod_path))
 225    )
 226    root_spec = (
 227        importlib.util.find_spec(root_name)
 228        if not root_path.exists()
 229        else importlib.util.spec_from_file_location(root_name, str(root_path))
 230    )
 231
 232    ### Check for updates before importing.
 233    _version = (
 234        determine_version(
 235            pathlib.Path(root_spec.origin),
 236            import_name=root_name, venv=venv, debug=debug
 237        ) if root_spec is not None and root_spec.origin is not None else None
 238    )
 239
 240    if _version is not None:
 241        if check_update:
 242            if need_update(
 243                None,
 244                import_name=root_name,
 245                version=_version,
 246                check_pypi=check_pypi,
 247                debug=debug,
 248            ):
 249                if install:
 250                    if not pip_install(
 251                        root_name,
 252                        venv=venv,
 253                        split=False,
 254                        check_update=check_update,
 255                        color=color,
 256                        debug=debug
 257                    ) and warn:
 258                        warn_function(
 259                            f"There's an update available for '{install_name}', "
 260                            + "but it failed to install. "
 261                            + "Try installig via Meerschaum with "
 262                            + "`install packages '{install_name}'`.",
 263                            ImportWarning,
 264                            stacklevel=3,
 265                            color=False,
 266                        )
 267                elif warn:
 268                    warn_function(
 269                        f"There's an update available for '{root_name}'.",
 270                        stack=False,
 271                        color=False,
 272                    )
 273                spec = (
 274                    importlib.util.find_spec(import_name)
 275                    if mod_path is None or not mod_path.exists()
 276                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
 277                )
 278
 279    if spec is None:
 280        try:
 281            mod = _import_module(import_name)
 282        except Exception:
 283            mod = None
 284        return mod
 285
 286    with Venv(venv, debug=debug):
 287        mod = importlib.util.module_from_spec(spec)
 288        old_sys_mod = sys.modules.get(import_name, None)
 289        sys.modules[import_name] = mod
 290
 291        try:
 292            with warnings.catch_warnings():
 293                warnings.filterwarnings('ignore', 'The NumPy')
 294                spec.loader.exec_module(mod)
 295        except Exception:
 296            pass
 297        mod = _import_module(import_name)
 298        if old_sys_mod is not None:
 299            sys.modules[import_name] = old_sys_mod
 300        else:
 301            del sys.modules[import_name]
 302
 303    return mod
 304
 305
 306def _import_to_install_name(import_name: str, with_version: bool = True) -> str:
 307    """
 308    Try to translate an import name to an installation name.
 309    """
 310    install_name = all_packages.get(import_name, import_name)
 311    if with_version:
 312        return install_name
 313    return get_install_no_version(install_name)
 314
 315
 316def _import_to_dir_name(import_name: str) -> str:
 317    """
 318    Translate an import name to the package name in the sites-packages directory.
 319    """
 320    import re
 321    return re.split(
 322        r'[<>=\[]', all_packages.get(import_name, import_name)
 323    )[0].replace('-', '_').lower() 
 324
 325
 326def _install_to_import_name(install_name: str) -> str:
 327    """
 328    Translate an installation name to a package's import name.
 329    """
 330    _install_no_version = get_install_no_version(install_name)
 331    return get_install_names().get(_install_no_version, _install_no_version)
 332
 333
 334def get_install_no_version(install_name: str) -> str:
 335    """
 336    Strip the version information from the install name.
 337    """
 338    import re
 339    return re.split(r'[\[=<>,! \]]', install_name)[0]
 340
 341
 342import_versions = {}
 343def determine_version(
 344    path: pathlib.Path,
 345    import_name: Optional[str] = None,
 346    venv: Optional[str] = 'mrsm',
 347    search_for_metadata: bool = True,
 348    split: bool = True,
 349    warn: bool = False,
 350    debug: bool = False,
 351) -> Union[str, None]:
 352    """
 353    Determine a module's `__version__` string from its filepath.
 354    
 355    First it searches for pip metadata, then it attempts to import the module in a subprocess.
 356
 357    Parameters
 358    ----------
 359    path: pathlib.Path
 360        The file path of the module.
 361
 362    import_name: Optional[str], default None
 363        The name of the module. If omitted, it will be determined from the file path.
 364        Defaults to `None`.
 365
 366    venv: Optional[str], default 'mrsm'
 367        The virtual environment of the Python interpreter to use if importing is necessary.
 368
 369    search_for_metadata: bool, default True
 370        If `True`, search the pip site_packages directory (assumed to be the parent)
 371        for the corresponding dist-info directory.
 372
 373    warn: bool, default True
 374        If `True`, raise a warning if the module fails to import in the subprocess.
 375
 376    split: bool, default True
 377        If `True`, split the determined import name by periods to get the room name.
 378
 379    Returns
 380    -------
 381    The package's version string if available or `None`.
 382    If multiple versions are found, it will trigger an import in a subprocess.
 383
 384    """
 385    with _locks['import_versions']:
 386        if venv not in import_versions:
 387            import_versions[venv] = {}
 388    import os
 389    old_cwd = os.getcwd()
 390    from meerschaum.utils.warnings import warn as warn_function
 391    if import_name is None:
 392        import_name = path.parent.stem if path.stem == '__init__' else path.stem
 393        import_name = import_name.split('.')[0] if split else import_name
 394    if import_name in import_versions[venv]:
 395        return import_versions[venv][import_name]
 396    _version = None
 397    module_parent_dir = (
 398        path.parent.parent if path.stem == '__init__' else path.parent
 399    ) if path is not None else venv_target_path(venv, allow_nonexistent=True, debug=debug)
 400
 401    if not module_parent_dir.exists():
 402        return None
 403
 404    installed_dir_name = _import_to_dir_name(import_name)
 405    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
 406
 407    ### First, check if a dist-info directory exists.
 408    _found_versions = []
 409    if search_for_metadata:
 410        try:
 411            filenames = os.listdir(module_parent_dir)
 412        except FileNotFoundError:
 413            filenames = []
 414        for filename in filenames:
 415            if not filename.endswith('.dist-info'):
 416                continue
 417            filename_lower = filename.lower()
 418            if not filename_lower.startswith(clean_installed_dir_name + '-'):
 419                continue
 420            _v = filename.replace('.dist-info', '').split("-")[-1]
 421            _found_versions.append(_v)
 422
 423    if len(_found_versions) == 1:
 424        _version = _found_versions[0]
 425        with _locks['import_versions']:
 426            import_versions[venv][import_name] = _version
 427        return _found_versions[0]
 428
 429    if not _found_versions:
 430        try:
 431            import importlib.metadata as importlib_metadata
 432        except ImportError:
 433            importlib_metadata = attempt_import(
 434                'importlib_metadata',
 435                debug=debug, check_update=False, precheck=False,
 436                color=False, check_is_installed=False, lazy=False,
 437            )
 438        try:
 439            os.chdir(module_parent_dir)
 440            _version = importlib_metadata.metadata(import_name)['Version']
 441        except Exception:
 442            _version = None
 443        finally:
 444            os.chdir(old_cwd)
 445
 446        if _version is not None:
 447            with _locks['import_versions']:
 448                import_versions[venv][import_name] = _version
 449            return _version
 450
 451    if debug:
 452        print(f'Found multiple versions for {import_name}: {_found_versions}')
 453
 454    module_parent_dir_str = module_parent_dir.as_posix()
 455
 456    ### Not a pip package, so let's try importing the module directly (in a subprocess).
 457    _no_version_str = 'no-version'
 458    code = (
 459        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
 460        + f"module = importlib.import_module('{import_name}');\n"
 461        + "try:\n"
 462        + "  print(module.__version__ , end='')\n"
 463        + "except:\n"
 464        + f"  print('{_no_version_str}', end='')"
 465    )
 466    exit_code, stdout_bytes, stderr_bytes = venv_exec(
 467        code, venv=venv, with_extras=True, debug=debug
 468    )
 469    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
 470    _version = stdout.split('\n')[-1] if exit_code == 0 else None
 471    _version = _version if _version != _no_version_str else None
 472
 473    if _version is None:
 474        _version = _get_package_metadata(import_name, venv).get('version', None)
 475    if _version is None and warn:
 476        warn_function(
 477            f"Failed to determine a version for '{import_name}':\n{stderr}",
 478            stack = False
 479        )
 480
 481    ### If `__version__` doesn't exist, return `None`.
 482    import_versions[venv][import_name] = _version
 483    return _version
 484
 485
 486def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]:
 487    """
 488    Get a package's metadata from pip.
 489    This is useful for getting a version when no `__version__` is defined
 490    and multiple versions are installed.
 491
 492    Parameters
 493    ----------
 494    import_name: str
 495        The package's import or installation name.
 496
 497    venv: Optional[str]
 498        The virtual environment which contains the package.
 499
 500    Returns
 501    -------
 502    A dictionary of metadata from pip.
 503    """
 504    import re
 505    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 506    install_name = _import_to_install_name(import_name)
 507    if install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX):
 508        return {}
 509    _args = ['pip', 'show', install_name]
 510    if venv is not None:
 511        cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 512        _args += ['--cache-dir', cache_dir_path.as_posix()]
 513
 514    if use_uv():
 515        package_name = 'uv'
 516        _args = ['pip', 'show', install_name]
 517    else:
 518        package_name = 'pip'
 519        _args = ['show', install_name]
 520
 521    proc = run_python_package(
 522        package_name, _args,
 523        capture_output=True, as_proc=True, venv=venv, universal_newlines=True,
 524    )
 525    outs, errs = proc.communicate()
 526    lines = outs.split('\n')
 527    meta = {}
 528    for line in lines:
 529        vals = line.split(": ")
 530        if len(vals) != 2:
 531            continue
 532        k, v = vals[0].lower(), vals[1]
 533        if v and 'UNKNOWN' not in v:
 534            meta[k] = v
 535    return meta
 536
 537
 538def need_update(
 539    package: Optional['ModuleType'] = None,
 540    install_name: Optional[str] = None,
 541    import_name: Optional[str] = None,
 542    version: Optional[str] = None,
 543    check_pypi: bool = False,
 544    split: bool = True,
 545    color: bool = True,
 546    debug: bool = False,
 547    _run_determine_version: bool = True,
 548) -> bool:
 549    """
 550    Check if a Meerschaum dependency needs an update.
 551    Returns a bool for whether or not a package needs to be updated.
 552
 553    Parameters
 554    ----------
 555    package: 'ModuleType'
 556        The module of the package to be updated.
 557
 558    install_name: Optional[str], default None
 559        If provided, use this string to determine the required version.
 560        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
 561
 562    import_name:
 563        If provided, override the package's `__name__` string.
 564
 565    version: Optional[str], default None
 566        If specified, override the package's `__version__` string.
 567
 568    check_pypi: bool, default False
 569        If `True`, check pypi.org for updates.
 570        Defaults to `False`.
 571
 572    split: bool, default True
 573        If `True`, split the module's name on periods to detrive the root name.
 574        Defaults to `True`.
 575
 576    color: bool, default True
 577        If `True`, format debug output.
 578        Defaults to `True`.
 579
 580    debug: bool, default True
 581        Verbosity toggle.
 582
 583    Returns
 584    -------
 585    A bool indicating whether the package requires an update.
 586
 587    """
 588    if debug:
 589        from meerschaum.utils.debug import dprint
 590    from meerschaum.utils.warnings import warn as warn_function
 591    import re
 592    root_name = (
 593        package.__name__.split('.')[0] if split else package.__name__
 594    ) if import_name is None else (
 595        import_name.split('.')[0] if split else import_name
 596    )
 597    install_name = install_name or _import_to_install_name(root_name)
 598    with _locks['_checked_for_updates']:
 599        if install_name in _checked_for_updates:
 600            return False
 601        _checked_for_updates.add(install_name)
 602
 603    _install_no_version = get_install_no_version(install_name)
 604    required_version = (
 605        install_name
 606        .replace(_install_no_version, '')
 607    )
 608    if ']' in required_version:
 609        required_version = required_version.split(']')[1]
 610
 611    ### No minimum version was specified, and we're not going to check PyPI.
 612    if not required_version and not check_pypi:
 613        return False
 614
 615    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
 616    if '.dev' in required_version:
 617        required_version = required_version.split('.dev')[0]
 618    if version and '.dev' in version:
 619        version = version.split('.dev')[0]
 620
 621    try:
 622        if not version:
 623            if not _run_determine_version:
 624                version = determine_version(
 625                    pathlib.Path(package.__file__),
 626                    import_name=root_name, warn=False, debug=debug
 627                )
 628        if version is None:
 629            return False
 630    except Exception as e:
 631        if debug:
 632            dprint(str(e), color=color)
 633            dprint("No version could be determined from the installed package.", color=color)
 634        return False
 635    split_version = version.split('.')
 636    last_part = split_version[-1]
 637    if len(split_version) == 2:
 638        version = '.'.join(split_version) + '.0'
 639    elif 'dev' in last_part or 'rc' in last_part:
 640        tag = 'dev' if 'dev' in last_part else 'rc'
 641        last_sep = '-'
 642        if not last_part.startswith(tag):
 643            last_part = f'-{tag}'.join(last_part.split(tag))
 644            last_sep = '.'
 645        version = '.'.join(split_version[:-1]) + last_sep + last_part
 646    elif len(split_version) > 3:
 647        version = '.'.join(split_version[:3])
 648
 649    packaging_version = attempt_import(
 650        'packaging.version', check_update=False, lazy=False, debug=debug,
 651    )
 652
 653    ### Get semver if necessary
 654    if required_version:
 655        semver_path = get_module_path('semver', debug=debug)
 656        if semver_path is None:
 657            no_venv_semver_path = get_module_path('semver', venv=None, debug=debug)
 658            if no_venv_semver_path is None:
 659                pip_install(_import_to_install_name('semver'), debug=debug)
 660        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
 661    if check_pypi:
 662        ### Check PyPI for updates
 663        update_checker = attempt_import(
 664            'update_checker', lazy=False, check_update=False, debug=debug
 665        )
 666        checker = update_checker.UpdateChecker()
 667        result = checker.check(_install_no_version, version)
 668    else:
 669        ### Skip PyPI and assume we can't be sure.
 670        result = None
 671
 672    ### Compare PyPI's version with our own.
 673    if result is not None:
 674        ### We have a result from PyPI and a stated required version.
 675        if required_version:
 676            try:
 677                return semver.Version.parse(result.available_version).match(required_version)
 678            except AttributeError as e:
 679                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 680                semver = manually_import_module('semver', venv='mrsm')
 681                return semver.Version.parse(version).match(required_version)
 682            except Exception as e:
 683                if debug:
 684                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
 685                return False
 686
 687        ### If `check_pypi` and we don't have a required version, check if PyPI's version
 688        ### is newer than the installed version.
 689        else:
 690            return (
 691                packaging_version.parse(result.available_version) > 
 692                packaging_version.parse(version)
 693            )
 694
 695    ### We might be depending on a prerelease.
 696    ### Sanity check that the required version is not greater than the installed version. 
 697    required_version = (
 698        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
 699        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
 700        .replace('-py3-none-any.whl', '')
 701    )
 702
 703    if 'a' in required_version:
 704        required_version = required_version.replace('a', '-pre.').replace('+mrsm', '')
 705        version = version.replace('a', '-pre.').replace('+mrsm', '')
 706    try:
 707        return (
 708            (not semver.Version.parse(version).match(required_version))
 709            if required_version else False
 710        )
 711    except AttributeError:
 712        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 713        semver = manually_import_module('semver', venv='mrsm', debug=debug)
 714        return (
 715            (not semver.Version.parse(version).match(required_version))
 716            if required_version else False
 717        )
 718    except Exception as e:
 719        print(f"Unable to parse version ({version}) for package '{import_name}'.")
 720        print(e)
 721        if debug:
 722            dprint(e)
 723        return False
 724    try:
 725        return (
 726            packaging_version.parse(version) > 
 727            packaging_version.parse(required_version)
 728        )
 729    except Exception as e:
 730        if debug:
 731            dprint(e)
 732        return False
 733    return False
 734
 735
 736def get_pip(
 737    venv: Optional[str] = 'mrsm',
 738    color: bool = True,
 739    debug: bool = False,
 740) -> bool:
 741    """
 742    Download and run the get-pip.py script.
 743
 744    Parameters
 745    ----------
 746    venv: Optional[str], default 'mrsm'
 747        The virtual environment into which to install `pip`.
 748
 749    color: bool, default True
 750        If `True`, force color output.
 751
 752    debug: bool, default False
 753        Verbosity toggle.
 754
 755    Returns
 756    -------
 757    A bool indicating success.
 758
 759    """
 760    import sys
 761    import subprocess
 762    from meerschaum.utils.misc import wget
 763    from meerschaum.config._paths import CACHE_RESOURCES_PATH
 764    from meerschaum._internal.static import STATIC_CONFIG
 765    url = STATIC_CONFIG['system']['urls']['get-pip.py']
 766    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
 767    try:
 768        wget(url, dest, color=False, debug=debug)
 769    except Exception:
 770        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
 771        sys.exit(1)
 772    if venv is not None:
 773        init_venv(venv=venv, debug=debug)
 774    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
 775    return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0
 776
 777
 778def pip_install(
 779    *install_names: str,
 780    args: Optional[List[str]] = None,
 781    requirements_file_path: Union[pathlib.Path, str, None] = None,
 782    venv: Optional[str] = 'mrsm',
 783    split: bool = False,
 784    check_update: bool = True,
 785    check_pypi: bool = True,
 786    check_wheel: bool = True,
 787    _uninstall: bool = False,
 788    _from_completely_uninstall: bool = False,
 789    _install_uv_pip: bool = True,
 790    _use_uv_pip: bool = True,
 791    color: bool = True,
 792    silent: bool = False,
 793    debug: bool = False,
 794) -> bool:
 795    """
 796    Install packages from PyPI with `pip`.
 797
 798    Parameters
 799    ----------
 800    *install_names: str
 801        The installation names of packages to be installed.
 802        This includes version restrictions.
 803        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 804        from its import name.
 805
 806    args: Optional[List[str]], default None
 807        A list of command line arguments to pass to `pip`.
 808        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 809
 810    requirements_file_path: Optional[pathlib.Path, str], default None
 811        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 812
 813    venv: str, default 'mrsm'
 814        The virtual environment to install into.
 815
 816    split: bool, default False
 817        If `True`, split on periods and only install the root package name.
 818
 819    check_update: bool, default True
 820        If `True`, check if the package requires an update.
 821
 822    check_pypi: bool, default True
 823        If `True` and `check_update` is `True`, check PyPI for the latest version.
 824
 825    check_wheel: bool, default True
 826        If `True`, check if `wheel` is available.
 827
 828    _uninstall: bool, default False
 829        If `True`, uninstall packages instead.
 830
 831    color: bool, default True
 832        If `True`, include color in debug text.
 833
 834    silent: bool, default False
 835        If `True`, skip printing messages.
 836
 837    debug: bool, default False
 838        Verbosity toggle.
 839
 840    Returns
 841    -------
 842    A bool indicating success.
 843
 844    """
 845    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 846    from meerschaum._internal.static import STATIC_CONFIG
 847    from meerschaum.utils.warnings import warn
 848    if args is None:
 849        args = ['--upgrade'] if not _uninstall else []
 850    ANSI = True if color else False
 851    if check_wheel:
 852        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 853
 854    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
 855    inside_daemon = daemon_env_var in os.environ
 856    if inside_daemon:
 857        silent = True
 858
 859    _args = list(args)
 860    have_pip = venv_contains_package('pip', venv=None, debug=debug)
 861    try:
 862        import pip
 863        have_pip = True
 864    except ImportError:
 865        have_pip = False
 866    try:
 867        import uv
 868        uv_bin = uv.find_uv_bin()
 869        have_uv_pip = True
 870    except (ImportError, FileNotFoundError):
 871        uv_bin = None
 872        have_uv_pip = False
 873    if have_pip and not have_uv_pip and _install_uv_pip and is_uv_enabled():
 874        if not pip_install(
 875            'uv',
 876            venv=None,
 877            debug=debug,
 878            _install_uv_pip=False,
 879            check_update=False,
 880            check_pypi=False,
 881            check_wheel=False,
 882        ) and not silent:
 883            warn(
 884                f"Failed to install `uv` for virtual environment '{venv}'.",
 885                color=False,
 886            )
 887
 888    use_uv_pip = (
 889        _use_uv_pip
 890        and venv_contains_package('uv', venv=None, debug=debug)
 891        and uv_bin is not None
 892        and venv is not None
 893        and is_uv_enabled()
 894    )
 895
 896    import sys
 897    if not have_pip and not use_uv_pip:
 898        if not get_pip(venv=venv, color=color, debug=debug):
 899            import sys
 900            minor = sys.version_info.minor
 901            print(
 902                "\nFailed to import `pip` and `ensurepip`.\n"
 903                + "If you are running Ubuntu/Debian, "
 904                + "you might need to install `python3.{minor}-distutils`:\n\n"
 905                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
 906                + "Please install pip and restart Meerschaum.\n\n"
 907                + "You can find instructions on installing `pip` here:\n"
 908                + "https://pip.pypa.io/en/stable/installing/"
 909            )
 910            sys.exit(1)
 911
 912    with Venv(venv, debug=debug):
 913        if venv is not None:
 914            if (
 915                '--ignore-installed' not in args
 916                and '-I' not in _args
 917                and not _uninstall
 918                and not use_uv_pip
 919            ):
 920                _args += ['--ignore-installed']
 921            if '--cache-dir' not in args and not _uninstall:
 922                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 923                _args += ['--cache-dir', str(cache_dir_path)]
 924
 925        if 'pip' not in ' '.join(_args) and not use_uv_pip:
 926            if check_update and not _uninstall:
 927                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
 928                if need_update(pip, check_pypi=check_pypi, debug=debug):
 929                    _args.append(all_packages['pip'])
 930
 931        _args = (['install'] if not _uninstall else ['uninstall']) + _args
 932
 933        if check_wheel and not _uninstall and not use_uv_pip:
 934            if not have_wheel:
 935                setup_packages_to_install = (
 936                    ['setuptools', 'wheel']
 937                    + (['uv'] if is_uv_enabled() else [])
 938                )
 939                if not pip_install(
 940                    *setup_packages_to_install,
 941                    venv=venv,
 942                    check_update=False,
 943                    check_pypi=False,
 944                    check_wheel=False,
 945                    debug=debug,
 946                    _install_uv_pip=False,
 947                ) and not silent:
 948                    from meerschaum.utils.misc import items_str
 949                    warn(
 950                        (
 951                            f"Failed to install {items_str(setup_packages_to_install)} for virtual "
 952                            + f"environment '{venv}'."
 953                        ),
 954                        color=False,
 955                    )
 956
 957        if requirements_file_path is not None:
 958            _args.append('-r')
 959            _args.append(pathlib.Path(requirements_file_path).resolve().as_posix())
 960
 961        if not ANSI and '--no-color' not in _args:
 962            _args.append('--no-color')
 963
 964        if '--no-input' not in _args and not use_uv_pip:
 965            _args.append('--no-input')
 966
 967        if _uninstall and '-y' not in _args and not use_uv_pip:
 968            _args.append('-y')
 969
 970        if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip:
 971            _args.append('--no-warn-conflicts')
 972
 973        if '--disable-pip-version-check' not in _args and not use_uv_pip:
 974            _args.append('--disable-pip-version-check')
 975
 976        if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
 977            if venv is not None:
 978                vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 979                if not vtp.exists():
 980                    if not init_venv(venv, force=True):
 981                        vtp.mkdir(parents=True, exist_ok=True)
 982                _args += ['--target', venv_target_path(venv, debug=debug)]
 983        elif (
 984            '--target' not in _args
 985                and '-t' not in _args
 986                and not inside_venv()
 987                and not _uninstall
 988                and not use_uv_pip
 989        ):
 990            _args += ['--user']
 991
 992        if debug:
 993            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
 994                if use_uv_pip:
 995                    _args.append('--verbose')
 996        else:
 997            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
 998                pass
 999
1000        _packages = [
1001            (
1002                get_install_no_version(install_name)
1003                if _uninstall or install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX)
1004                else install_name
1005            )
1006            for install_name in install_names
1007        ]
1008        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
1009        for p in _packages:
1010            msg += f'\n  - {p}'
1011        if not silent:
1012            print(msg)
1013
1014        if _uninstall and not _from_completely_uninstall and not use_uv_pip:
1015            for install_name in _packages:
1016                _install_no_version = get_install_no_version(install_name)
1017                if _install_no_version in ('pip', 'wheel', 'uv'):
1018                    continue
1019                if not completely_uninstall_package(
1020                    _install_no_version,
1021                    venv=venv, debug=debug,
1022                ) and not silent:
1023                    warn(
1024                        f"Failed to clean up package '{_install_no_version}'.",
1025                    )
1026
1027        ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease.
1028        if use_uv_pip:
1029            _args.insert(0, 'pip')
1030            if not _uninstall and get_prerelease_dependencies(_packages):
1031                _args.append('--prerelease=allow')
1032
1033        rc = run_python_package(
1034            ('pip' if not use_uv_pip else 'uv'),
1035            _args + _packages,
1036            venv=None,
1037            env=_get_pip_os_env(color=color),
1038            debug=debug,
1039        )
1040        if debug:
1041            print(f"{rc=}")
1042        success = rc == 0
1043
1044    msg = (
1045        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
1046        else "Failed to " + ('un' if _uninstall else '') + "install packages."
1047    )
1048    if not silent:
1049        print(msg)
1050    if debug and not silent:
1051        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
1052    return success
1053
1054
1055def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1056    """
1057    Return a list of explicitly prerelease dependencies from a list of packages.
1058    """
1059    if _packages is None:
1060        _packages = list(all_packages.keys())
1061    prelrease_strings = ['dev', 'rc', 'a']
1062    prerelease_packages = []
1063    for install_name in _packages:
1064        _install_no_version = get_install_no_version(install_name)
1065        import_name = _install_to_import_name(install_name)
1066        install_with_version = _import_to_install_name(import_name)
1067        version_only = (
1068            install_with_version.lower().replace(_install_no_version.lower(), '')
1069            .split(']')[-1]
1070        )
1071
1072        is_prerelease = False
1073        for prelrease_string in prelrease_strings:
1074            if prelrease_string in version_only:
1075                is_prerelease = True
1076
1077        if is_prerelease:
1078            prerelease_packages.append(install_name)
1079    return prerelease_packages
1080
1081
1082def completely_uninstall_package(
1083    install_name: str,
1084    venv: str = 'mrsm',
1085    debug: bool = False,
1086) -> bool:
1087    """
1088    Continue calling `pip uninstall` until a package is completely
1089    removed from a virtual environment. 
1090    This is useful for dealing with multiple installed versions of a package.
1091    """
1092    attempts = 0
1093    _install_no_version = get_install_no_version(install_name)
1094    clean_install_no_version = _install_no_version.lower().replace('-', '_')
1095    installed_versions = []
1096    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1097    if not vtp.exists():
1098        return True
1099
1100    for file_name in os.listdir(vtp):
1101        if not file_name.endswith('.dist-info'):
1102            continue
1103        clean_dist_info = file_name.replace('-', '_').lower()
1104        if not clean_dist_info.startswith(clean_install_no_version):
1105            continue
1106        installed_versions.append(file_name)
1107
1108    max_attempts = len(installed_versions)
1109    while attempts < max_attempts:
1110        if not venv_contains_package(
1111            _install_to_import_name(_install_no_version),
1112            venv=venv, debug=debug,
1113        ):
1114            return True
1115        if not pip_uninstall(
1116            _install_no_version,
1117            venv = venv,
1118            silent = (not debug),
1119            _from_completely_uninstall = True,
1120            debug = debug,
1121        ):
1122            return False
1123        attempts += 1
1124    return False
1125
1126
1127def pip_uninstall(
1128    *args, **kw
1129) -> bool:
1130    """
1131    Uninstall Python packages.
1132    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
1133    """
1134    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
1135
1136
1137def run_python_package(
1138    package_name: str,
1139    args: Optional[List[str]] = None,
1140    venv: Optional[str] = 'mrsm',
1141    cwd: Optional[str] = None,
1142    foreground: bool = False,
1143    as_proc: bool = False,
1144    capture_output: bool = False,
1145    debug: bool = False,
1146    **kw: Any,
1147) -> Union[int, subprocess.Popen, None]:
1148    """
1149    Runs an installed python package.
1150    E.g. Translates to `/usr/bin/python -m [package]`
1151
1152    Parameters
1153    ----------
1154    package_name: str
1155        The Python module to be executed.
1156
1157    args: Optional[List[str]], default None
1158        Additional command line arguments to be appended after `-m [package]`.
1159
1160    venv: Optional[str], default 'mrsm'
1161        If specified, execute the Python interpreter from a virtual environment.
1162
1163    cwd: Optional[str], default None
1164        If specified, change directories before starting the process.
1165        Defaults to `None`.
1166
1167    as_proc: bool, default False
1168        If `True`, return a `subprocess.Popen` object.
1169
1170    capture_output: bool, default False
1171        If `as_proc` is `True`, capture stdout and stderr.
1172
1173    foreground: bool, default False
1174        If `True`, start the subprocess as a foreground process.
1175        Defaults to `False`.
1176
1177    kw: Any
1178        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1179        and by extension `subprocess.Popen()`.
1180
1181    Returns
1182    -------
1183    Either a return code integer or a `subprocess.Popen` object
1184    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1185    """
1186    import sys
1187    import platform
1188    import subprocess
1189    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1190    from meerschaum.utils.process import run_process
1191    from meerschaum.utils.warnings import warn
1192    if args is None:
1193        args = []
1194    old_cwd = os.getcwd()
1195    if cwd is not None:
1196        os.chdir(cwd)
1197    executable = venv_executable(venv=venv)
1198    venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
1199    env_dict = kw.get('env', os.environ).copy()
1200    if venv_path is not None:
1201        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
1202    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1203    import traceback
1204    if debug:
1205        print(command, file=sys.stderr)
1206    try:
1207        to_return = run_process(
1208            command,
1209            foreground=foreground,
1210            as_proc=as_proc,
1211            capture_output=capture_output,
1212            **kw
1213        )
1214    except Exception:
1215        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1216        warn(msg, color=False)
1217        stdout, stderr = (
1218            (None, None)
1219            if not capture_output
1220            else (subprocess.PIPE, subprocess.PIPE)
1221        )
1222        proc = subprocess.Popen(
1223            command,
1224            stdout=stdout,
1225            stderr=stderr,
1226            stdin=sys.stdin,
1227            env=env_dict,
1228        )
1229        to_return = proc if as_proc else proc.wait()
1230    except KeyboardInterrupt:
1231        to_return = 1 if not as_proc else None
1232    os.chdir(old_cwd)
1233    return to_return
1234
1235
1236def attempt_import(
1237    *names: str,
1238    lazy: bool = True,
1239    warn: bool = True,
1240    install: bool = True,
1241    venv: Optional[str] = 'mrsm',
1242    precheck: bool = True,
1243    split: bool = True,
1244    check_update: bool = False,
1245    check_pypi: bool = False,
1246    check_is_installed: bool = True,
1247    allow_outside_venv: bool = True,
1248    color: bool = True,
1249    debug: bool = False
1250) -> Any:
1251    """
1252    Raise a warning if packages are not installed; otherwise import and return modules.
1253    If `lazy` is `True`, return lazy-imported modules.
1254    
1255    Returns tuple of modules if multiple names are provided, else returns one module.
1256    
1257    Parameters
1258    ----------
1259    names: List[str]
1260        The packages to be imported.
1261
1262    lazy: bool, default True
1263        If `True`, lazily load packages.
1264
1265    warn: bool, default True
1266        If `True`, raise a warning if a package cannot be imported.
1267
1268    install: bool, default True
1269        If `True`, attempt to install a missing package into the designated virtual environment.
1270        If `check_update` is True, install updates if available.
1271
1272    venv: Optional[str], default 'mrsm'
1273        The virtual environment in which to search for packages and to install packages into.
1274
1275    precheck: bool, default True
1276        If `True`, attempt to find module before importing (necessary for checking if modules exist
1277        and retaining lazy imports), otherwise assume lazy is `False`.
1278
1279    split: bool, default True
1280        If `True`, split packages' names on `'.'`.
1281
1282    check_update: bool, default False
1283        If `True` and `install` is `True`, install updates if the required minimum version
1284        does not match.
1285
1286    check_pypi: bool, default False
1287        If `True` and `check_update` is `True`, check PyPI when determining whether
1288        an update is required.
1289
1290    check_is_installed: bool, default True
1291        If `True`, check if the package is contained in the virtual environment.
1292
1293    allow_outside_venv: bool, default True
1294        If `True`, search outside of the specified virtual environment
1295        if the package cannot be found.
1296        Setting to `False` will reinstall the package into a virtual environment, even if it
1297        is installed outside.
1298
1299    color: bool, default True
1300        If `False`, do not print ANSI colors.
1301
1302    Returns
1303    -------
1304    The specified modules. If they're not available and `install` is `True`, it will first
1305    download them into a virtual environment and return the modules.
1306
1307    Examples
1308    --------
1309    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1310    >>> pandas = attempt_import('pandas')
1311
1312    """
1313
1314    import importlib.util
1315
1316    ### to prevent recursion, check if parent Meerschaum package is being imported
1317    if names == ('meerschaum',):
1318        return _import_module('meerschaum')
1319
1320    if venv == 'mrsm' and _import_hook_venv is not None:
1321        if debug:
1322            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1323        venv = _import_hook_venv
1324
1325    _warnings = _import_module('meerschaum.utils.warnings')
1326    warn_function = _warnings.warn
1327
1328    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1329        with Venv(venv=venv, debug=debug):
1330            ### determine the import method (lazy vs normal)
1331            from meerschaum.utils.misc import filter_keywords
1332            import_method = (
1333                _import_module if not lazy
1334                else lazy_import
1335            )
1336            try:
1337                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1338            except Exception as e:
1339                if warn:
1340                    import traceback
1341                    traceback.print_exception(type(e), e, e.__traceback__)
1342                    warn_function(
1343                        f"Failed to import module '{_name}'.\nException:\n{e}",
1344                        ImportWarning,
1345                        stacklevel = (5 if lazy else 4),
1346                        color = False,
1347                    )
1348                mod = None
1349        return mod
1350
1351    modules = []
1352    for name in names:
1353        ### Check if package is a declared dependency.
1354        root_name = name.split('.')[0] if split else name
1355        install_name = _import_to_install_name(root_name)
1356
1357        if install_name is None:
1358            install_name = root_name
1359            if warn and root_name != 'plugins':
1360                warn_function(
1361                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1362                    ImportWarning,
1363                    stacklevel = 3,
1364                    color = False
1365                )
1366
1367        ### Determine if the package exists.
1368        if precheck is False:
1369            found_module = (
1370                do_import(
1371                    name, debug=debug, warn=False, venv=venv, color=color,
1372                    check_update=False, check_pypi=False, split=split,
1373                ) is not None
1374            )
1375        else:
1376            if check_is_installed:
1377                with _locks['_is_installed_first_check']:
1378                    if not _is_installed_first_check.get(name, False):
1379                        package_is_installed = is_installed(
1380                            name,
1381                            venv = venv,
1382                            split = split,
1383                            allow_outside_venv = allow_outside_venv,
1384                            debug = debug,
1385                        )
1386                        _is_installed_first_check[name] = package_is_installed
1387                    else:
1388                        package_is_installed = _is_installed_first_check[name]
1389            else:
1390                package_is_installed = _is_installed_first_check.get(
1391                    name,
1392                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1393                )
1394            found_module = package_is_installed
1395
1396        if not found_module:
1397            if install:
1398                if not pip_install(
1399                    install_name,
1400                    venv = venv,
1401                    split = False,
1402                    check_update = check_update,
1403                    color = color,
1404                    debug = debug
1405                ) and warn:
1406                    warn_function(
1407                        f"Failed to install '{install_name}'.",
1408                        ImportWarning,
1409                        stacklevel = 3,
1410                        color = False,
1411                    )
1412            elif warn:
1413                ### Raise a warning if we can't find the package and install = False.
1414                warn_function(
1415                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1416                     + "some features will not work correctly."
1417                     + "\n\nSet install=True when calling attempt_import.\n"),
1418                    ImportWarning,
1419                    stacklevel = 3,
1420                    color = False,
1421                )
1422
1423        ### Do the import. Will be lazy if lazy=True.
1424        m = do_import(
1425            name, debug=debug, warn=warn, venv=venv, color=color,
1426            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1427        )
1428        modules.append(m)
1429
1430    modules = tuple(modules)
1431    if len(modules) == 1:
1432        return modules[0]
1433    return modules
1434
1435
1436def lazy_import(
1437    name: str,
1438    local_name: str = None,
1439    **kw
1440) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1441    """
1442    Lazily import a package.
1443    """
1444    from meerschaum.utils.packages.lazy_loader import LazyLoader
1445    if local_name is None:
1446        local_name = name
1447    return LazyLoader(
1448        local_name,
1449        globals(),
1450        name,
1451        **kw
1452    )
1453
1454
1455def pandas_name() -> str:
1456    """
1457    Return the configured name for `pandas`.
1458    
1459    Below are the expected possible values:
1460
1461    - 'pandas'
1462    - 'modin.pandas'
1463    - 'dask.dataframe'
1464
1465    """
1466    from meerschaum.config import get_config
1467    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1468    if pandas_module_name == 'modin':
1469        pandas_module_name = 'modin.pandas'
1470    elif pandas_module_name == 'dask':
1471        pandas_module_name = 'dask.dataframe'
1472
1473    return pandas_module_name
1474
1475
1476emitted_pandas_warning: bool = False
1477def import_pandas(
1478    debug: bool = False,
1479    lazy: bool = False,
1480    **kw
1481) -> 'ModuleType':
1482    """
1483    Quality-of-life function to attempt to import the configured version of `pandas`.
1484    """
1485    pandas_module_name = pandas_name()
1486    global emitted_pandas_warning
1487
1488    if pandas_module_name != 'pandas':
1489        with _locks['emitted_pandas_warning']:
1490            if not emitted_pandas_warning:
1491                from meerschaum.utils.warnings import warn
1492                emitted_pandas_warning = True
1493                warn(
1494                    (
1495                        "You are using an alternative Pandas implementation "
1496                        + f"'{pandas_module_name}'"
1497                        + "\n   Features may not work as expected."
1498                    ),
1499                    stack=False,
1500                )
1501
1502    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1503    pandas, pyarrow = attempt_import('pandas', 'pyarrow', debug=debug, lazy=False, **kw)
1504    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1505    return pd
1506
1507
1508def import_rich(
1509    lazy: bool = True,
1510    debug: bool = False,
1511    **kw: Any
1512) -> 'ModuleType':
1513    """
1514    Quality of life function for importing `rich`.
1515    """
1516    from meerschaum.utils.formatting import ANSI, UNICODE
1517    ## need typing_extensions for `from rich import box`
1518    typing_extensions = attempt_import(
1519        'typing_extensions', lazy=False, debug=debug
1520    )
1521    pygments = attempt_import(
1522        'pygments', lazy=False,
1523    )
1524    rich = attempt_import(
1525        'rich', lazy=lazy,
1526        **kw
1527    )
1528    return rich
1529
1530
1531def _dash_less_than_2(**kw) -> bool:
1532    dash = attempt_import('dash', **kw)
1533    if dash is None:
1534        return None
1535    packaging_version = attempt_import('packaging.version', **kw)
1536    return (
1537        packaging_version.parse(dash.__version__) < 
1538        packaging_version.parse('2.0.0')
1539    )
1540
1541
1542def import_dcc(warn=False, **kw) -> 'ModuleType':
1543    """
1544    Import Dash Core Components (`dcc`).
1545    """
1546    return (
1547        attempt_import('dash_core_components', warn=warn, **kw)
1548        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1549    )
1550
1551
1552def import_html(warn=False, **kw) -> 'ModuleType':
1553    """
1554    Import Dash HTML Components (`html`).
1555    """
1556    return (
1557        attempt_import('dash_html_components', warn=warn, **kw)
1558        if _dash_less_than_2(warn=warn, **kw)
1559        else attempt_import('dash.html', warn=warn, **kw)
1560    )
1561
1562
1563def get_modules_from_package(
1564    package: 'package',
1565    names: bool = False,
1566    recursive: bool = False,
1567    lazy: bool = False,
1568    modules_venvs: bool = False,
1569    debug: bool = False
1570):
1571    """
1572    Find and import all modules in a package.
1573    
1574    Returns
1575    -------
1576    Either list of modules or tuple of lists.
1577    """
1578    from os.path import dirname, join, isfile, isdir, basename
1579    import glob
1580
1581    pattern = '*' if recursive else '*.py'
1582    package_path = dirname(package.__file__ or package.__path__[0])
1583    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1584    _all = [
1585        basename(f)[:-3] if isfile(f) else basename(f)
1586        for f in module_names
1587            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1588               and not f.endswith('__init__.py')
1589               and not f.endswith('__pycache__')
1590    ]
1591
1592    if debug:
1593        from meerschaum.utils.debug import dprint
1594        dprint(str(_all))
1595    modules = []
1596    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1597        ### there's probably a better way than a try: catch but it'll do for now
1598        try:
1599            ### if specified, activate the module's virtual environment before importing.
1600            ### NOTE: this only considers the filename, so two modules from different packages
1601            ### may end up sharing virtual environments.
1602            if modules_venvs:
1603                activate_venv(module_name.split('.')[-1], debug=debug)
1604            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1605            modules.append(m)
1606        except Exception as e:
1607            if debug:
1608                dprint(str(e))
1609        finally:
1610            if modules_venvs:
1611                deactivate_venv(module_name.split('.')[-1], debug=debug)
1612    if names:
1613        return _all, modules
1614
1615    return modules
1616
1617
1618def import_children(
1619    package: Optional['ModuleType'] = None,
1620    package_name: Optional[str] = None,
1621    types : Optional[List[str]] = None,
1622    lazy: bool = True,
1623    recursive: bool = False,
1624    debug: bool = False
1625) -> List['ModuleType']:
1626    """
1627    Import all functions in a package to its `__init__`.
1628
1629    Parameters
1630    ----------
1631    package: Optional[ModuleType], default None
1632        Package to import its functions into.
1633        If `None` (default), use parent.
1634
1635    package_name: Optional[str], default None
1636        Name of package to import its functions into
1637        If None (default), use parent.
1638
1639    types: Optional[List[str]], default None
1640        Types of members to return.
1641        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1642
1643    Returns
1644    -------
1645    A list of modules.
1646    """
1647    import sys, inspect
1648
1649    if types is None:
1650        types = ['method', 'builtin', 'function', 'class', 'module']
1651
1652    ### if package_name and package are None, use parent
1653    if package is None and package_name is None:
1654        package_name = inspect.stack()[1][0].f_globals['__name__']
1655
1656    ### populate package or package_name from other other
1657    if package is None:
1658        package = sys.modules[package_name]
1659    elif package_name is None:
1660        package_name = package.__name__
1661
1662    ### Set attributes in sys module version of package.
1663    ### Kinda like setting a dictionary
1664    ###   functions[name] = func
1665    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1666    _all, members = [], []
1667    objects = []
1668    for module in modules:
1669        _objects = []
1670        for ob in inspect.getmembers(module):
1671            for t in types:
1672                ### ob is a tuple of (name, object)
1673                if getattr(inspect, 'is' + t)(ob[1]):
1674                    _objects.append(ob)
1675
1676        if 'module' in types:
1677            _objects.append((module.__name__.split('.')[0], module))
1678        objects += _objects
1679    for ob in objects:
1680        setattr(sys.modules[package_name], ob[0], ob[1])
1681        _all.append(ob[0])
1682        members.append(ob[1])
1683
1684    if debug:
1685        from meerschaum.utils.debug import dprint
1686        dprint(str(_all))
1687    ### set __all__ for import *
1688    setattr(sys.modules[package_name], '__all__', _all)
1689    return members
1690
1691
1692_reload_module_cache = {}
1693def reload_package(
1694    package: str,
1695    skip_submodules: Optional[List[str]] = None,
1696    lazy: bool = False,
1697    debug: bool = False,
1698    **kw: Any
1699):
1700    """
1701    Recursively load a package's subpackages, even if they were not previously loaded.
1702    """
1703    import sys
1704    if isinstance(package, str):
1705        package_name = package
1706    else:
1707        try:
1708            package_name = package.__name__
1709        except Exception as e:
1710            package_name = str(package)
1711
1712    skip_submodules = skip_submodules or []
1713    if 'meerschaum.utils.packages' not in skip_submodules:
1714        skip_submodules.append('meerschaum.utils.packages')
1715    def safeimport():
1716        subs = [
1717            m for m in sys.modules
1718            if m.startswith(package_name + '.')
1719        ]
1720        subs_to_skip = []
1721        for skip_mod in skip_submodules:
1722            for mod in subs:
1723                if mod.startswith(skip_mod):
1724                    subs_to_skip.append(mod)
1725                    continue
1726
1727        subs = [m for m in subs if m not in subs_to_skip]
1728        for module_name in subs:
1729            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1730        if not subs_to_skip:
1731            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1732
1733        return _import_module(package_name)
1734
1735    return safeimport()
1736
1737
1738def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1739    """
1740    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1741    """
1742    reload_package(
1743        'meerschaum',
1744        skip_submodules = [
1745            'meerschaum._internal.shell',
1746            'meerschaum.utils.pool',
1747        ]
1748    )
1749
1750    from meerschaum.plugins import reload_plugins
1751    from meerschaum._internal.shell.Shell import _insert_shell_actions
1752    reload_plugins(debug=debug)
1753    _insert_shell_actions()
1754    return True, "Success"
1755
1756
1757def is_installed(
1758    import_name: str,
1759    venv: Optional[str] = 'mrsm',
1760    split: bool = True,
1761    allow_outside_venv: bool = True,
1762    debug: bool = False,
1763) -> bool:
1764    """
1765    Check whether a package is installed.
1766
1767    Parameters
1768    ----------
1769    import_name: str
1770        The import name of the module.
1771
1772    venv: Optional[str], default 'mrsm'
1773        The venv in which to search for the module.
1774
1775    split: bool, default True
1776        If `True`, split on periods to determine the root module name.
1777
1778    allow_outside_venv: bool, default True
1779        If `True`, search outside of the specified virtual environment
1780        if the package cannot be found.
1781
1782    Returns
1783    -------
1784    A bool indicating whether a package may be imported.
1785    """
1786    if debug:
1787        from meerschaum.utils.debug import dprint
1788    root_name = import_name.split('.')[0] if split else import_name
1789    import importlib.util
1790    with Venv(venv, debug=debug):
1791        try:
1792            spec_path = pathlib.Path(
1793                get_module_path(root_name, venv=venv, debug=debug)
1794                or
1795                (
1796                    importlib.util.find_spec(root_name).origin 
1797                    if venv is not None and allow_outside_venv
1798                    else None
1799                )
1800            )
1801        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1802            spec_path = None
1803
1804        found = (
1805            not need_update(
1806                None,
1807                import_name=root_name,
1808                _run_determine_version=False,
1809                check_pypi=False,
1810                version=determine_version(
1811                    spec_path,
1812                    venv=venv,
1813                    debug=debug,
1814                    import_name=root_name,
1815                ),
1816                debug=debug,
1817            )
1818        ) if spec_path is not None else False
1819
1820    return found
1821
1822
1823def venv_contains_package(
1824    import_name: str,
1825    venv: Optional[str] = 'mrsm',
1826    split: bool = True,
1827    debug: bool = False,
1828) -> bool:
1829    """
1830    Search the contents of a virtual environment for a package.
1831    """
1832    import site
1833    import pathlib
1834    root_name = import_name.split('.')[0] if split else import_name
1835    return get_module_path(root_name, venv=venv, debug=debug) is not None
1836
1837
1838def package_venv(package: 'ModuleType') -> Union[str, None]:
1839    """
1840    Inspect a package and return the virtual environment in which it presides.
1841    """
1842    import os
1843    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1844    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1845        return None
1846    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
1847
1848
1849def ensure_readline() -> 'ModuleType':
1850    """Make sure that the `readline` package is able to be imported."""
1851    import sys
1852    try:
1853        import readline
1854    except ImportError:
1855        readline = None
1856
1857    if readline is None:
1858        import platform
1859        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1860        try:
1861            rl = attempt_import(
1862                rl_name,
1863                lazy=False,
1864                install=True,
1865                venv=None,
1866                warn=False,
1867            )
1868        except (ImportError, ModuleNotFoundError):
1869            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1870                print(f"Unable to import {rl_name}!", file=sys.stderr)
1871                sys.exit(1)
1872
1873    sys.modules['readline'] = readline
1874    return readline
1875
1876
1877def _get_pip_os_env(color: bool = True):
1878    """
1879    Return the environment variables context in which `pip` should be run.
1880    See PEP 668 for why we are overriding the environment.
1881    """
1882    import os, sys, platform
1883    python_bin_path = pathlib.Path(sys.executable)
1884    pip_os_env = os.environ.copy()
1885    path_str = pip_os_env.get('PATH', '') or ''
1886    path_sep = ':' if platform.system() != 'Windows' else ';'
1887    pip_os_env.update({
1888        'PIP_BREAK_SYSTEM_PACKAGES': 'true',
1889        'UV_BREAK_SYSTEM_PACKAGES': 'true',
1890        ('FORCE_COLOR' if color else 'NO_COLOR'): '1',
1891    })
1892    if str(python_bin_path) not in path_str:
1893        pip_os_env['PATH'] = str(python_bin_path.parent) + path_sep + path_str
1894
1895    return pip_os_env
1896
1897
1898def use_uv() -> bool:
1899    """
1900    Return whether `uv` is available and enabled.
1901    """
1902    from meerschaum.utils.misc import is_android
1903    if is_android():
1904        return False
1905
1906    if not is_uv_enabled():
1907        return False
1908
1909    try:
1910        import uv
1911        uv_bin = uv.find_uv_bin()
1912    except (ImportError, FileNotFoundError):
1913        uv_bin = None
1914
1915    if uv_bin is None:
1916        return False
1917
1918    return True
1919
1920
1921def is_uv_enabled() -> bool:
1922    """
1923    Return whether the user has disabled `uv`.
1924    """
1925    from meerschaum.utils.misc import is_android
1926    if is_android():
1927        return False
1928
1929    from meerschaum.utils.venv import inside_venv
1930
1931    if inside_venv():
1932        return False
1933
1934    try:
1935        import yaml
1936    except ImportError:
1937        return False
1938
1939    from meerschaum.config import get_config
1940    enabled = get_config('system', 'experimental', 'uv_pip')
1941    return enabled
def get_module_path( import_name: str, venv: Optional[str] = 'mrsm', debug: bool = False, _try_install_name_on_fail: bool = True) -> Optional[pathlib.Path]:
 46def get_module_path(
 47    import_name: str,
 48    venv: Optional[str] = 'mrsm',
 49    debug: bool = False,
 50    _try_install_name_on_fail: bool = True,
 51) -> Union[pathlib.Path, None]:
 52    """
 53    Get a module's path without importing.
 54    """
 55    import site
 56    if debug:
 57        from meerschaum.utils.debug import dprint
 58    if not _try_install_name_on_fail:
 59        install_name = _import_to_install_name(import_name, with_version=False)
 60        install_name_lower = install_name.lower().replace('-', '_')
 61        import_name_lower = install_name_lower
 62    else:
 63        import_name_lower = import_name.lower().replace('-', '_')
 64
 65    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 66    if not vtp.exists():
 67        if debug:
 68            dprint(
 69                (
 70                    "Venv '{venv}' does not exist, cannot import "
 71                    + f"'{import_name}'."
 72                ),
 73                color = False,
 74            )
 75        return None
 76
 77    venv_target_candidate_paths = [vtp]
 78    if venv is None:
 79        site_user_packages_dirs = [
 80            pathlib.Path(site.getusersitepackages())
 81        ] if not inside_venv() else []
 82        site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
 83
 84        paths_to_add = [
 85            path
 86            for path in site_user_packages_dirs + site_packages_dirs
 87            if path not in venv_target_candidate_paths
 88        ]
 89        venv_target_candidate_paths += paths_to_add
 90
 91    candidates = []
 92    for venv_target_candidate in venv_target_candidate_paths:
 93        try:
 94            file_names = os.listdir(venv_target_candidate)
 95        except FileNotFoundError:
 96            continue
 97        for file_name in file_names:
 98            file_name_lower = file_name.lower().replace('-', '_')
 99            if not file_name_lower.startswith(import_name_lower):
100                continue
101            if file_name.endswith('dist_info'):
102                continue
103            file_path = venv_target_candidate / file_name
104
105            ### Most likely: Is a directory with __init__.py
106            if file_name_lower == import_name_lower and file_path.is_dir():
107                init_path = file_path / '__init__.py'
108                if init_path.exists():
109                    candidates.append(init_path)
110
111            ### May be a standalone .py file.
112            elif file_name_lower == import_name_lower + '.py':
113                candidates.append(file_path)
114
115            ### Compiled wheels (e.g. pyodbc)
116            elif file_name_lower.startswith(import_name_lower + '.'):
117                candidates.append(file_path)
118
119    if len(candidates) == 1:
120        return candidates[0]
121
122    if not candidates:
123        if _try_install_name_on_fail:
124            return get_module_path(
125                import_name, venv=venv, debug=debug,
126                _try_install_name_on_fail=False
127            )
128        return None
129
130    specs_paths = []
131    for candidate_path in candidates:
132        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
133        if spec is not None:
134            return candidate_path
135    
136    return None

Get a module's path without importing.

def manually_import_module( import_name: str, venv: Optional[str] = 'mrsm', check_update: bool = True, check_pypi: bool = False, install: bool = True, split: bool = True, warn: bool = True, color: bool = True, debug: bool = False, use_sys_modules: bool = True) -> "Union['ModuleType', None]":
139def manually_import_module(
140    import_name: str,
141    venv: Optional[str] = 'mrsm',
142    check_update: bool = True,
143    check_pypi: bool = False,
144    install: bool = True,
145    split: bool = True,
146    warn: bool = True,
147    color: bool = True,
148    debug: bool = False,
149    use_sys_modules: bool = True,
150) -> Union['ModuleType', None]:
151    """
152    Manually import a module from a virtual environment (or the base environment).
153
154    Parameters
155    ----------
156    import_name: str
157        The name of the module.
158        
159    venv: Optional[str], default 'mrsm'
160        The virtual environment to read from.
161
162    check_update: bool, default True
163        If `True`, examine whether the available version of the package meets the required version.
164
165    check_pypi: bool, default False
166        If `True`, check PyPI for updates before importing.
167
168    install: bool, default True
169        If `True`, install the package if it's not installed or needs an update.
170
171    split: bool, default True
172        If `True`, split `import_name` on periods to get the package name.
173
174    warn: bool, default True
175        If `True`, raise a warning if the package cannot be imported.
176
177    color: bool, default True
178        If `True`, use color output for debug and warning text.
179
180    debug: bool, default False
181        Verbosity toggle.
182
183    use_sys_modules: bool, default True
184        If `True`, return the module in `sys.modules` if it exists.
185        Otherwise continue with manually importing.
186
187    Returns
188    -------
189    The specified module or `None` if it can't be imported.
190
191    """
192    import sys
193    _previously_imported = import_name in sys.modules
194    if _previously_imported and use_sys_modules:
195        return sys.modules[import_name]
196
197    from meerschaum.utils.warnings import warn as warn_function
198    import warnings
199    root_name = import_name.split('.')[0] if split else import_name
200    install_name = _import_to_install_name(root_name)
201
202    root_path = get_module_path(root_name, venv=venv)
203    if root_path is None:
204        return None
205
206    mod_path = root_path
207    if mod_path.is_dir():
208        for _dir in import_name.split('.')[:-1]:
209            mod_path = mod_path / _dir
210            possible_end_module_filename = import_name.split('.')[-1] + '.py'
211            try:
212                mod_path = (
213                    (mod_path / possible_end_module_filename)
214                    if possible_end_module_filename in os.listdir(mod_path)
215                    else (
216                        mod_path / import_name.split('.')[-1] / '__init__.py'
217                    )
218                )
219            except Exception:
220                mod_path = None
221
222    spec = (
223        importlib.util.find_spec(import_name)
224        if mod_path is None or not mod_path.exists()
225        else importlib.util.spec_from_file_location(import_name, str(mod_path))
226    )
227    root_spec = (
228        importlib.util.find_spec(root_name)
229        if not root_path.exists()
230        else importlib.util.spec_from_file_location(root_name, str(root_path))
231    )
232
233    ### Check for updates before importing.
234    _version = (
235        determine_version(
236            pathlib.Path(root_spec.origin),
237            import_name=root_name, venv=venv, debug=debug
238        ) if root_spec is not None and root_spec.origin is not None else None
239    )
240
241    if _version is not None:
242        if check_update:
243            if need_update(
244                None,
245                import_name=root_name,
246                version=_version,
247                check_pypi=check_pypi,
248                debug=debug,
249            ):
250                if install:
251                    if not pip_install(
252                        root_name,
253                        venv=venv,
254                        split=False,
255                        check_update=check_update,
256                        color=color,
257                        debug=debug
258                    ) and warn:
259                        warn_function(
260                            f"There's an update available for '{install_name}', "
261                            + "but it failed to install. "
262                            + "Try installig via Meerschaum with "
263                            + "`install packages '{install_name}'`.",
264                            ImportWarning,
265                            stacklevel=3,
266                            color=False,
267                        )
268                elif warn:
269                    warn_function(
270                        f"There's an update available for '{root_name}'.",
271                        stack=False,
272                        color=False,
273                    )
274                spec = (
275                    importlib.util.find_spec(import_name)
276                    if mod_path is None or not mod_path.exists()
277                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
278                )
279
280    if spec is None:
281        try:
282            mod = _import_module(import_name)
283        except Exception:
284            mod = None
285        return mod
286
287    with Venv(venv, debug=debug):
288        mod = importlib.util.module_from_spec(spec)
289        old_sys_mod = sys.modules.get(import_name, None)
290        sys.modules[import_name] = mod
291
292        try:
293            with warnings.catch_warnings():
294                warnings.filterwarnings('ignore', 'The NumPy')
295                spec.loader.exec_module(mod)
296        except Exception:
297            pass
298        mod = _import_module(import_name)
299        if old_sys_mod is not None:
300            sys.modules[import_name] = old_sys_mod
301        else:
302            del sys.modules[import_name]
303
304    return mod

Manually import a module from a virtual environment (or the base environment).

Parameters
  • import_name (str): The name of the module.
  • venv (Optional[str], default 'mrsm'): The virtual environment to read from.
  • check_update (bool, default True): If True, examine whether the available version of the package meets the required version.
  • check_pypi (bool, default False): If True, check PyPI for updates before importing.
  • install (bool, default True): If True, install the package if it's not installed or needs an update.
  • split (bool, default True): If True, split import_name on periods to get the package name.
  • warn (bool, default True): If True, raise a warning if the package cannot be imported.
  • color (bool, default True): If True, use color output for debug and warning text.
  • debug (bool, default False): Verbosity toggle.
  • use_sys_modules (bool, default True): If True, return the module in sys.modules if it exists. Otherwise continue with manually importing.
Returns
  • The specified module or None if it can't be imported.
def get_install_no_version(install_name: str) -> str:
335def get_install_no_version(install_name: str) -> str:
336    """
337    Strip the version information from the install name.
338    """
339    import re
340    return re.split(r'[\[=<>,! \]]', install_name)[0]

Strip the version information from the install name.

import_versions = {'mrsm': {'daemon': '3.1.2', 'packaging': '25.0', 'semver': '3.0.4', 'pandas': '2.3.1', 'prompt_toolkit': '3.0.51', 'dask': '2025.7.0'}}
def determine_version( path: pathlib.Path, import_name: Optional[str] = None, venv: Optional[str] = 'mrsm', search_for_metadata: bool = True, split: bool = True, warn: bool = False, debug: bool = False) -> Optional[str]:
344def determine_version(
345    path: pathlib.Path,
346    import_name: Optional[str] = None,
347    venv: Optional[str] = 'mrsm',
348    search_for_metadata: bool = True,
349    split: bool = True,
350    warn: bool = False,
351    debug: bool = False,
352) -> Union[str, None]:
353    """
354    Determine a module's `__version__` string from its filepath.
355    
356    First it searches for pip metadata, then it attempts to import the module in a subprocess.
357
358    Parameters
359    ----------
360    path: pathlib.Path
361        The file path of the module.
362
363    import_name: Optional[str], default None
364        The name of the module. If omitted, it will be determined from the file path.
365        Defaults to `None`.
366
367    venv: Optional[str], default 'mrsm'
368        The virtual environment of the Python interpreter to use if importing is necessary.
369
370    search_for_metadata: bool, default True
371        If `True`, search the pip site_packages directory (assumed to be the parent)
372        for the corresponding dist-info directory.
373
374    warn: bool, default True
375        If `True`, raise a warning if the module fails to import in the subprocess.
376
377    split: bool, default True
378        If `True`, split the determined import name by periods to get the room name.
379
380    Returns
381    -------
382    The package's version string if available or `None`.
383    If multiple versions are found, it will trigger an import in a subprocess.
384
385    """
386    with _locks['import_versions']:
387        if venv not in import_versions:
388            import_versions[venv] = {}
389    import os
390    old_cwd = os.getcwd()
391    from meerschaum.utils.warnings import warn as warn_function
392    if import_name is None:
393        import_name = path.parent.stem if path.stem == '__init__' else path.stem
394        import_name = import_name.split('.')[0] if split else import_name
395    if import_name in import_versions[venv]:
396        return import_versions[venv][import_name]
397    _version = None
398    module_parent_dir = (
399        path.parent.parent if path.stem == '__init__' else path.parent
400    ) if path is not None else venv_target_path(venv, allow_nonexistent=True, debug=debug)
401
402    if not module_parent_dir.exists():
403        return None
404
405    installed_dir_name = _import_to_dir_name(import_name)
406    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
407
408    ### First, check if a dist-info directory exists.
409    _found_versions = []
410    if search_for_metadata:
411        try:
412            filenames = os.listdir(module_parent_dir)
413        except FileNotFoundError:
414            filenames = []
415        for filename in filenames:
416            if not filename.endswith('.dist-info'):
417                continue
418            filename_lower = filename.lower()
419            if not filename_lower.startswith(clean_installed_dir_name + '-'):
420                continue
421            _v = filename.replace('.dist-info', '').split("-")[-1]
422            _found_versions.append(_v)
423
424    if len(_found_versions) == 1:
425        _version = _found_versions[0]
426        with _locks['import_versions']:
427            import_versions[venv][import_name] = _version
428        return _found_versions[0]
429
430    if not _found_versions:
431        try:
432            import importlib.metadata as importlib_metadata
433        except ImportError:
434            importlib_metadata = attempt_import(
435                'importlib_metadata',
436                debug=debug, check_update=False, precheck=False,
437                color=False, check_is_installed=False, lazy=False,
438            )
439        try:
440            os.chdir(module_parent_dir)
441            _version = importlib_metadata.metadata(import_name)['Version']
442        except Exception:
443            _version = None
444        finally:
445            os.chdir(old_cwd)
446
447        if _version is not None:
448            with _locks['import_versions']:
449                import_versions[venv][import_name] = _version
450            return _version
451
452    if debug:
453        print(f'Found multiple versions for {import_name}: {_found_versions}')
454
455    module_parent_dir_str = module_parent_dir.as_posix()
456
457    ### Not a pip package, so let's try importing the module directly (in a subprocess).
458    _no_version_str = 'no-version'
459    code = (
460        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
461        + f"module = importlib.import_module('{import_name}');\n"
462        + "try:\n"
463        + "  print(module.__version__ , end='')\n"
464        + "except:\n"
465        + f"  print('{_no_version_str}', end='')"
466    )
467    exit_code, stdout_bytes, stderr_bytes = venv_exec(
468        code, venv=venv, with_extras=True, debug=debug
469    )
470    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
471    _version = stdout.split('\n')[-1] if exit_code == 0 else None
472    _version = _version if _version != _no_version_str else None
473
474    if _version is None:
475        _version = _get_package_metadata(import_name, venv).get('version', None)
476    if _version is None and warn:
477        warn_function(
478            f"Failed to determine a version for '{import_name}':\n{stderr}",
479            stack = False
480        )
481
482    ### If `__version__` doesn't exist, return `None`.
483    import_versions[venv][import_name] = _version
484    return _version

Determine a module's __version__ string from its filepath.

First it searches for pip metadata, then it attempts to import the module in a subprocess.

Parameters
  • path (pathlib.Path): The file path of the module.
  • import_name (Optional[str], default None): The name of the module. If omitted, it will be determined from the file path. Defaults to None.
  • venv (Optional[str], default 'mrsm'): The virtual environment of the Python interpreter to use if importing is necessary.
  • search_for_metadata (bool, default True): If True, search the pip site_packages directory (assumed to be the parent) for the corresponding dist-info directory.
  • warn (bool, default True): If True, raise a warning if the module fails to import in the subprocess.
  • split (bool, default True): If True, split the determined import name by periods to get the room name.
Returns
  • The package's version string if available or None.
  • If multiple versions are found, it will trigger an import in a subprocess.
def need_update( package: "Optional['ModuleType']" = None, install_name: Optional[str] = None, import_name: Optional[str] = None, version: Optional[str] = None, check_pypi: bool = False, split: bool = True, color: bool = True, debug: bool = False, _run_determine_version: bool = True) -> bool:
539def need_update(
540    package: Optional['ModuleType'] = None,
541    install_name: Optional[str] = None,
542    import_name: Optional[str] = None,
543    version: Optional[str] = None,
544    check_pypi: bool = False,
545    split: bool = True,
546    color: bool = True,
547    debug: bool = False,
548    _run_determine_version: bool = True,
549) -> bool:
550    """
551    Check if a Meerschaum dependency needs an update.
552    Returns a bool for whether or not a package needs to be updated.
553
554    Parameters
555    ----------
556    package: 'ModuleType'
557        The module of the package to be updated.
558
559    install_name: Optional[str], default None
560        If provided, use this string to determine the required version.
561        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
562
563    import_name:
564        If provided, override the package's `__name__` string.
565
566    version: Optional[str], default None
567        If specified, override the package's `__version__` string.
568
569    check_pypi: bool, default False
570        If `True`, check pypi.org for updates.
571        Defaults to `False`.
572
573    split: bool, default True
574        If `True`, split the module's name on periods to detrive the root name.
575        Defaults to `True`.
576
577    color: bool, default True
578        If `True`, format debug output.
579        Defaults to `True`.
580
581    debug: bool, default True
582        Verbosity toggle.
583
584    Returns
585    -------
586    A bool indicating whether the package requires an update.
587
588    """
589    if debug:
590        from meerschaum.utils.debug import dprint
591    from meerschaum.utils.warnings import warn as warn_function
592    import re
593    root_name = (
594        package.__name__.split('.')[0] if split else package.__name__
595    ) if import_name is None else (
596        import_name.split('.')[0] if split else import_name
597    )
598    install_name = install_name or _import_to_install_name(root_name)
599    with _locks['_checked_for_updates']:
600        if install_name in _checked_for_updates:
601            return False
602        _checked_for_updates.add(install_name)
603
604    _install_no_version = get_install_no_version(install_name)
605    required_version = (
606        install_name
607        .replace(_install_no_version, '')
608    )
609    if ']' in required_version:
610        required_version = required_version.split(']')[1]
611
612    ### No minimum version was specified, and we're not going to check PyPI.
613    if not required_version and not check_pypi:
614        return False
615
616    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
617    if '.dev' in required_version:
618        required_version = required_version.split('.dev')[0]
619    if version and '.dev' in version:
620        version = version.split('.dev')[0]
621
622    try:
623        if not version:
624            if not _run_determine_version:
625                version = determine_version(
626                    pathlib.Path(package.__file__),
627                    import_name=root_name, warn=False, debug=debug
628                )
629        if version is None:
630            return False
631    except Exception as e:
632        if debug:
633            dprint(str(e), color=color)
634            dprint("No version could be determined from the installed package.", color=color)
635        return False
636    split_version = version.split('.')
637    last_part = split_version[-1]
638    if len(split_version) == 2:
639        version = '.'.join(split_version) + '.0'
640    elif 'dev' in last_part or 'rc' in last_part:
641        tag = 'dev' if 'dev' in last_part else 'rc'
642        last_sep = '-'
643        if not last_part.startswith(tag):
644            last_part = f'-{tag}'.join(last_part.split(tag))
645            last_sep = '.'
646        version = '.'.join(split_version[:-1]) + last_sep + last_part
647    elif len(split_version) > 3:
648        version = '.'.join(split_version[:3])
649
650    packaging_version = attempt_import(
651        'packaging.version', check_update=False, lazy=False, debug=debug,
652    )
653
654    ### Get semver if necessary
655    if required_version:
656        semver_path = get_module_path('semver', debug=debug)
657        if semver_path is None:
658            no_venv_semver_path = get_module_path('semver', venv=None, debug=debug)
659            if no_venv_semver_path is None:
660                pip_install(_import_to_install_name('semver'), debug=debug)
661        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
662    if check_pypi:
663        ### Check PyPI for updates
664        update_checker = attempt_import(
665            'update_checker', lazy=False, check_update=False, debug=debug
666        )
667        checker = update_checker.UpdateChecker()
668        result = checker.check(_install_no_version, version)
669    else:
670        ### Skip PyPI and assume we can't be sure.
671        result = None
672
673    ### Compare PyPI's version with our own.
674    if result is not None:
675        ### We have a result from PyPI and a stated required version.
676        if required_version:
677            try:
678                return semver.Version.parse(result.available_version).match(required_version)
679            except AttributeError as e:
680                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
681                semver = manually_import_module('semver', venv='mrsm')
682                return semver.Version.parse(version).match(required_version)
683            except Exception as e:
684                if debug:
685                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
686                return False
687
688        ### If `check_pypi` and we don't have a required version, check if PyPI's version
689        ### is newer than the installed version.
690        else:
691            return (
692                packaging_version.parse(result.available_version) > 
693                packaging_version.parse(version)
694            )
695
696    ### We might be depending on a prerelease.
697    ### Sanity check that the required version is not greater than the installed version. 
698    required_version = (
699        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
700        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
701        .replace('-py3-none-any.whl', '')
702    )
703
704    if 'a' in required_version:
705        required_version = required_version.replace('a', '-pre.').replace('+mrsm', '')
706        version = version.replace('a', '-pre.').replace('+mrsm', '')
707    try:
708        return (
709            (not semver.Version.parse(version).match(required_version))
710            if required_version else False
711        )
712    except AttributeError:
713        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
714        semver = manually_import_module('semver', venv='mrsm', debug=debug)
715        return (
716            (not semver.Version.parse(version).match(required_version))
717            if required_version else False
718        )
719    except Exception as e:
720        print(f"Unable to parse version ({version}) for package '{import_name}'.")
721        print(e)
722        if debug:
723            dprint(e)
724        return False
725    try:
726        return (
727            packaging_version.parse(version) > 
728            packaging_version.parse(required_version)
729        )
730    except Exception as e:
731        if debug:
732            dprint(e)
733        return False
734    return False

Check if a Meerschaum dependency needs an update. Returns a bool for whether or not a package needs to be updated.

Parameters
  • package ('ModuleType'): The module of the package to be updated.
  • install_name (Optional[str], default None): If provided, use this string to determine the required version. Otherwise use the install name defined in meerschaum.utils.packages._packages.
  • import_name:: If provided, override the package's __name__ string.
  • version (Optional[str], default None): If specified, override the package's __version__ string.
  • check_pypi (bool, default False): If True, check pypi.org for updates. Defaults to False.
  • split (bool, default True): If True, split the module's name on periods to detrive the root name. Defaults to True.
  • color (bool, default True): If True, format debug output. Defaults to True.
  • debug (bool, default True): Verbosity toggle.
Returns
  • A bool indicating whether the package requires an update.
def get_pip( venv: Optional[str] = 'mrsm', color: bool = True, debug: bool = False) -> bool:
737def get_pip(
738    venv: Optional[str] = 'mrsm',
739    color: bool = True,
740    debug: bool = False,
741) -> bool:
742    """
743    Download and run the get-pip.py script.
744
745    Parameters
746    ----------
747    venv: Optional[str], default 'mrsm'
748        The virtual environment into which to install `pip`.
749
750    color: bool, default True
751        If `True`, force color output.
752
753    debug: bool, default False
754        Verbosity toggle.
755
756    Returns
757    -------
758    A bool indicating success.
759
760    """
761    import sys
762    import subprocess
763    from meerschaum.utils.misc import wget
764    from meerschaum.config._paths import CACHE_RESOURCES_PATH
765    from meerschaum._internal.static import STATIC_CONFIG
766    url = STATIC_CONFIG['system']['urls']['get-pip.py']
767    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
768    try:
769        wget(url, dest, color=False, debug=debug)
770    except Exception:
771        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
772        sys.exit(1)
773    if venv is not None:
774        init_venv(venv=venv, debug=debug)
775    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
776    return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0

Download and run the get-pip.py script.

Parameters
  • venv (Optional[str], default 'mrsm'): The virtual environment into which to install pip.
  • color (bool, default True): If True, force color output.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating success.
def pip_install( *install_names: str, args: Optional[List[str]] = None, requirements_file_path: Union[pathlib.Path, str, NoneType] = None, venv: Optional[str] = 'mrsm', split: bool = False, check_update: bool = True, check_pypi: bool = True, check_wheel: bool = True, _uninstall: bool = False, _from_completely_uninstall: bool = False, _install_uv_pip: bool = True, _use_uv_pip: bool = True, color: bool = True, silent: bool = False, debug: bool = False) -> bool:
 779def pip_install(
 780    *install_names: str,
 781    args: Optional[List[str]] = None,
 782    requirements_file_path: Union[pathlib.Path, str, None] = None,
 783    venv: Optional[str] = 'mrsm',
 784    split: bool = False,
 785    check_update: bool = True,
 786    check_pypi: bool = True,
 787    check_wheel: bool = True,
 788    _uninstall: bool = False,
 789    _from_completely_uninstall: bool = False,
 790    _install_uv_pip: bool = True,
 791    _use_uv_pip: bool = True,
 792    color: bool = True,
 793    silent: bool = False,
 794    debug: bool = False,
 795) -> bool:
 796    """
 797    Install packages from PyPI with `pip`.
 798
 799    Parameters
 800    ----------
 801    *install_names: str
 802        The installation names of packages to be installed.
 803        This includes version restrictions.
 804        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 805        from its import name.
 806
 807    args: Optional[List[str]], default None
 808        A list of command line arguments to pass to `pip`.
 809        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 810
 811    requirements_file_path: Optional[pathlib.Path, str], default None
 812        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 813
 814    venv: str, default 'mrsm'
 815        The virtual environment to install into.
 816
 817    split: bool, default False
 818        If `True`, split on periods and only install the root package name.
 819
 820    check_update: bool, default True
 821        If `True`, check if the package requires an update.
 822
 823    check_pypi: bool, default True
 824        If `True` and `check_update` is `True`, check PyPI for the latest version.
 825
 826    check_wheel: bool, default True
 827        If `True`, check if `wheel` is available.
 828
 829    _uninstall: bool, default False
 830        If `True`, uninstall packages instead.
 831
 832    color: bool, default True
 833        If `True`, include color in debug text.
 834
 835    silent: bool, default False
 836        If `True`, skip printing messages.
 837
 838    debug: bool, default False
 839        Verbosity toggle.
 840
 841    Returns
 842    -------
 843    A bool indicating success.
 844
 845    """
 846    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 847    from meerschaum._internal.static import STATIC_CONFIG
 848    from meerschaum.utils.warnings import warn
 849    if args is None:
 850        args = ['--upgrade'] if not _uninstall else []
 851    ANSI = True if color else False
 852    if check_wheel:
 853        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 854
 855    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
 856    inside_daemon = daemon_env_var in os.environ
 857    if inside_daemon:
 858        silent = True
 859
 860    _args = list(args)
 861    have_pip = venv_contains_package('pip', venv=None, debug=debug)
 862    try:
 863        import pip
 864        have_pip = True
 865    except ImportError:
 866        have_pip = False
 867    try:
 868        import uv
 869        uv_bin = uv.find_uv_bin()
 870        have_uv_pip = True
 871    except (ImportError, FileNotFoundError):
 872        uv_bin = None
 873        have_uv_pip = False
 874    if have_pip and not have_uv_pip and _install_uv_pip and is_uv_enabled():
 875        if not pip_install(
 876            'uv',
 877            venv=None,
 878            debug=debug,
 879            _install_uv_pip=False,
 880            check_update=False,
 881            check_pypi=False,
 882            check_wheel=False,
 883        ) and not silent:
 884            warn(
 885                f"Failed to install `uv` for virtual environment '{venv}'.",
 886                color=False,
 887            )
 888
 889    use_uv_pip = (
 890        _use_uv_pip
 891        and venv_contains_package('uv', venv=None, debug=debug)
 892        and uv_bin is not None
 893        and venv is not None
 894        and is_uv_enabled()
 895    )
 896
 897    import sys
 898    if not have_pip and not use_uv_pip:
 899        if not get_pip(venv=venv, color=color, debug=debug):
 900            import sys
 901            minor = sys.version_info.minor
 902            print(
 903                "\nFailed to import `pip` and `ensurepip`.\n"
 904                + "If you are running Ubuntu/Debian, "
 905                + "you might need to install `python3.{minor}-distutils`:\n\n"
 906                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
 907                + "Please install pip and restart Meerschaum.\n\n"
 908                + "You can find instructions on installing `pip` here:\n"
 909                + "https://pip.pypa.io/en/stable/installing/"
 910            )
 911            sys.exit(1)
 912
 913    with Venv(venv, debug=debug):
 914        if venv is not None:
 915            if (
 916                '--ignore-installed' not in args
 917                and '-I' not in _args
 918                and not _uninstall
 919                and not use_uv_pip
 920            ):
 921                _args += ['--ignore-installed']
 922            if '--cache-dir' not in args and not _uninstall:
 923                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 924                _args += ['--cache-dir', str(cache_dir_path)]
 925
 926        if 'pip' not in ' '.join(_args) and not use_uv_pip:
 927            if check_update and not _uninstall:
 928                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
 929                if need_update(pip, check_pypi=check_pypi, debug=debug):
 930                    _args.append(all_packages['pip'])
 931
 932        _args = (['install'] if not _uninstall else ['uninstall']) + _args
 933
 934        if check_wheel and not _uninstall and not use_uv_pip:
 935            if not have_wheel:
 936                setup_packages_to_install = (
 937                    ['setuptools', 'wheel']
 938                    + (['uv'] if is_uv_enabled() else [])
 939                )
 940                if not pip_install(
 941                    *setup_packages_to_install,
 942                    venv=venv,
 943                    check_update=False,
 944                    check_pypi=False,
 945                    check_wheel=False,
 946                    debug=debug,
 947                    _install_uv_pip=False,
 948                ) and not silent:
 949                    from meerschaum.utils.misc import items_str
 950                    warn(
 951                        (
 952                            f"Failed to install {items_str(setup_packages_to_install)} for virtual "
 953                            + f"environment '{venv}'."
 954                        ),
 955                        color=False,
 956                    )
 957
 958        if requirements_file_path is not None:
 959            _args.append('-r')
 960            _args.append(pathlib.Path(requirements_file_path).resolve().as_posix())
 961
 962        if not ANSI and '--no-color' not in _args:
 963            _args.append('--no-color')
 964
 965        if '--no-input' not in _args and not use_uv_pip:
 966            _args.append('--no-input')
 967
 968        if _uninstall and '-y' not in _args and not use_uv_pip:
 969            _args.append('-y')
 970
 971        if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip:
 972            _args.append('--no-warn-conflicts')
 973
 974        if '--disable-pip-version-check' not in _args and not use_uv_pip:
 975            _args.append('--disable-pip-version-check')
 976
 977        if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
 978            if venv is not None:
 979                vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 980                if not vtp.exists():
 981                    if not init_venv(venv, force=True):
 982                        vtp.mkdir(parents=True, exist_ok=True)
 983                _args += ['--target', venv_target_path(venv, debug=debug)]
 984        elif (
 985            '--target' not in _args
 986                and '-t' not in _args
 987                and not inside_venv()
 988                and not _uninstall
 989                and not use_uv_pip
 990        ):
 991            _args += ['--user']
 992
 993        if debug:
 994            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
 995                if use_uv_pip:
 996                    _args.append('--verbose')
 997        else:
 998            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
 999                pass
1000
1001        _packages = [
1002            (
1003                get_install_no_version(install_name)
1004                if _uninstall or install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX)
1005                else install_name
1006            )
1007            for install_name in install_names
1008        ]
1009        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
1010        for p in _packages:
1011            msg += f'\n  - {p}'
1012        if not silent:
1013            print(msg)
1014
1015        if _uninstall and not _from_completely_uninstall and not use_uv_pip:
1016            for install_name in _packages:
1017                _install_no_version = get_install_no_version(install_name)
1018                if _install_no_version in ('pip', 'wheel', 'uv'):
1019                    continue
1020                if not completely_uninstall_package(
1021                    _install_no_version,
1022                    venv=venv, debug=debug,
1023                ) and not silent:
1024                    warn(
1025                        f"Failed to clean up package '{_install_no_version}'.",
1026                    )
1027
1028        ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease.
1029        if use_uv_pip:
1030            _args.insert(0, 'pip')
1031            if not _uninstall and get_prerelease_dependencies(_packages):
1032                _args.append('--prerelease=allow')
1033
1034        rc = run_python_package(
1035            ('pip' if not use_uv_pip else 'uv'),
1036            _args + _packages,
1037            venv=None,
1038            env=_get_pip_os_env(color=color),
1039            debug=debug,
1040        )
1041        if debug:
1042            print(f"{rc=}")
1043        success = rc == 0
1044
1045    msg = (
1046        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
1047        else "Failed to " + ('un' if _uninstall else '') + "install packages."
1048    )
1049    if not silent:
1050        print(msg)
1051    if debug and not silent:
1052        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
1053    return success

Install packages from PyPI with pip.

Parameters
  • *install_names (str): The installation names of packages to be installed. This includes version restrictions. Use _import_to_install_name() to get the predefined install_name for a package from its import name.
  • args (Optional[List[str]], default None): A list of command line arguments to pass to pip. If not provided, default to ['--upgrade'] if _uninstall is False, else [].
  • requirements_file_path (Optional[pathlib.Path, str], default None): If provided, append ['-r', '/path/to/requirements.txt'] to args.
  • venv (str, default 'mrsm'): The virtual environment to install into.
  • split (bool, default False): If True, split on periods and only install the root package name.
  • check_update (bool, default True): If True, check if the package requires an update.
  • check_pypi (bool, default True): If True and check_update is True, check PyPI for the latest version.
  • check_wheel (bool, default True): If True, check if wheel is available.
  • _uninstall (bool, default False): If True, uninstall packages instead.
  • color (bool, default True): If True, include color in debug text.
  • silent (bool, default False): If True, skip printing messages.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating success.
def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1056def get_prerelease_dependencies(_packages: Optional[List[str]] = None):
1057    """
1058    Return a list of explicitly prerelease dependencies from a list of packages.
1059    """
1060    if _packages is None:
1061        _packages = list(all_packages.keys())
1062    prelrease_strings = ['dev', 'rc', 'a']
1063    prerelease_packages = []
1064    for install_name in _packages:
1065        _install_no_version = get_install_no_version(install_name)
1066        import_name = _install_to_import_name(install_name)
1067        install_with_version = _import_to_install_name(import_name)
1068        version_only = (
1069            install_with_version.lower().replace(_install_no_version.lower(), '')
1070            .split(']')[-1]
1071        )
1072
1073        is_prerelease = False
1074        for prelrease_string in prelrease_strings:
1075            if prelrease_string in version_only:
1076                is_prerelease = True
1077
1078        if is_prerelease:
1079            prerelease_packages.append(install_name)
1080    return prerelease_packages

Return a list of explicitly prerelease dependencies from a list of packages.

def completely_uninstall_package(install_name: str, venv: str = 'mrsm', debug: bool = False) -> bool:
1083def completely_uninstall_package(
1084    install_name: str,
1085    venv: str = 'mrsm',
1086    debug: bool = False,
1087) -> bool:
1088    """
1089    Continue calling `pip uninstall` until a package is completely
1090    removed from a virtual environment. 
1091    This is useful for dealing with multiple installed versions of a package.
1092    """
1093    attempts = 0
1094    _install_no_version = get_install_no_version(install_name)
1095    clean_install_no_version = _install_no_version.lower().replace('-', '_')
1096    installed_versions = []
1097    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
1098    if not vtp.exists():
1099        return True
1100
1101    for file_name in os.listdir(vtp):
1102        if not file_name.endswith('.dist-info'):
1103            continue
1104        clean_dist_info = file_name.replace('-', '_').lower()
1105        if not clean_dist_info.startswith(clean_install_no_version):
1106            continue
1107        installed_versions.append(file_name)
1108
1109    max_attempts = len(installed_versions)
1110    while attempts < max_attempts:
1111        if not venv_contains_package(
1112            _install_to_import_name(_install_no_version),
1113            venv=venv, debug=debug,
1114        ):
1115            return True
1116        if not pip_uninstall(
1117            _install_no_version,
1118            venv = venv,
1119            silent = (not debug),
1120            _from_completely_uninstall = True,
1121            debug = debug,
1122        ):
1123            return False
1124        attempts += 1
1125    return False

Continue calling pip uninstall until a package is completely removed from a virtual environment. This is useful for dealing with multiple installed versions of a package.

def pip_uninstall(*args, **kw) -> bool:
1128def pip_uninstall(
1129    *args, **kw
1130) -> bool:
1131    """
1132    Uninstall Python packages.
1133    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
1134    """
1135    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})

Uninstall Python packages. This function is a wrapper around pip_install() but with _uninstall enforced as True.

def run_python_package( package_name: str, args: Optional[List[str]] = None, venv: Optional[str] = 'mrsm', cwd: Optional[str] = None, foreground: bool = False, as_proc: bool = False, capture_output: bool = False, debug: bool = False, **kw: Any) -> Union[int, subprocess.Popen, NoneType]:
1138def run_python_package(
1139    package_name: str,
1140    args: Optional[List[str]] = None,
1141    venv: Optional[str] = 'mrsm',
1142    cwd: Optional[str] = None,
1143    foreground: bool = False,
1144    as_proc: bool = False,
1145    capture_output: bool = False,
1146    debug: bool = False,
1147    **kw: Any,
1148) -> Union[int, subprocess.Popen, None]:
1149    """
1150    Runs an installed python package.
1151    E.g. Translates to `/usr/bin/python -m [package]`
1152
1153    Parameters
1154    ----------
1155    package_name: str
1156        The Python module to be executed.
1157
1158    args: Optional[List[str]], default None
1159        Additional command line arguments to be appended after `-m [package]`.
1160
1161    venv: Optional[str], default 'mrsm'
1162        If specified, execute the Python interpreter from a virtual environment.
1163
1164    cwd: Optional[str], default None
1165        If specified, change directories before starting the process.
1166        Defaults to `None`.
1167
1168    as_proc: bool, default False
1169        If `True`, return a `subprocess.Popen` object.
1170
1171    capture_output: bool, default False
1172        If `as_proc` is `True`, capture stdout and stderr.
1173
1174    foreground: bool, default False
1175        If `True`, start the subprocess as a foreground process.
1176        Defaults to `False`.
1177
1178    kw: Any
1179        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1180        and by extension `subprocess.Popen()`.
1181
1182    Returns
1183    -------
1184    Either a return code integer or a `subprocess.Popen` object
1185    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1186    """
1187    import sys
1188    import platform
1189    import subprocess
1190    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1191    from meerschaum.utils.process import run_process
1192    from meerschaum.utils.warnings import warn
1193    if args is None:
1194        args = []
1195    old_cwd = os.getcwd()
1196    if cwd is not None:
1197        os.chdir(cwd)
1198    executable = venv_executable(venv=venv)
1199    venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None
1200    env_dict = kw.get('env', os.environ).copy()
1201    if venv_path is not None:
1202        env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()})
1203    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1204    import traceback
1205    if debug:
1206        print(command, file=sys.stderr)
1207    try:
1208        to_return = run_process(
1209            command,
1210            foreground=foreground,
1211            as_proc=as_proc,
1212            capture_output=capture_output,
1213            **kw
1214        )
1215    except Exception:
1216        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1217        warn(msg, color=False)
1218        stdout, stderr = (
1219            (None, None)
1220            if not capture_output
1221            else (subprocess.PIPE, subprocess.PIPE)
1222        )
1223        proc = subprocess.Popen(
1224            command,
1225            stdout=stdout,
1226            stderr=stderr,
1227            stdin=sys.stdin,
1228            env=env_dict,
1229        )
1230        to_return = proc if as_proc else proc.wait()
1231    except KeyboardInterrupt:
1232        to_return = 1 if not as_proc else None
1233    os.chdir(old_cwd)
1234    return to_return

Runs an installed python package. E.g. Translates to /usr/bin/python -m [package]

Parameters
  • package_name (str): The Python module to be executed.
  • args (Optional[List[str]], default None): Additional command line arguments to be appended after -m [package].
  • venv (Optional[str], default 'mrsm'): If specified, execute the Python interpreter from a virtual environment.
  • cwd (Optional[str], default None): If specified, change directories before starting the process. Defaults to None.
  • as_proc (bool, default False): If True, return a subprocess.Popen object.
  • capture_output (bool, default False): If as_proc is True, capture stdout and stderr.
  • foreground (bool, default False): If True, start the subprocess as a foreground process. Defaults to False.
  • kw (Any): Additional keyword arguments to pass to meerschaum.utils.process.run_process() and by extension subprocess.Popen().
Returns
  • Either a return code integer or a subprocess.Popen object
  • (or None if a KeyboardInterrupt occurs and as_proc is True).
def attempt_import( *names: str, lazy: bool = True, warn: bool = True, install: bool = True, venv: Optional[str] = 'mrsm', precheck: bool = True, split: bool = True, check_update: bool = False, check_pypi: bool = False, check_is_installed: bool = True, allow_outside_venv: bool = True, color: bool = True, debug: bool = False) -> Any:
1237def attempt_import(
1238    *names: str,
1239    lazy: bool = True,
1240    warn: bool = True,
1241    install: bool = True,
1242    venv: Optional[str] = 'mrsm',
1243    precheck: bool = True,
1244    split: bool = True,
1245    check_update: bool = False,
1246    check_pypi: bool = False,
1247    check_is_installed: bool = True,
1248    allow_outside_venv: bool = True,
1249    color: bool = True,
1250    debug: bool = False
1251) -> Any:
1252    """
1253    Raise a warning if packages are not installed; otherwise import and return modules.
1254    If `lazy` is `True`, return lazy-imported modules.
1255    
1256    Returns tuple of modules if multiple names are provided, else returns one module.
1257    
1258    Parameters
1259    ----------
1260    names: List[str]
1261        The packages to be imported.
1262
1263    lazy: bool, default True
1264        If `True`, lazily load packages.
1265
1266    warn: bool, default True
1267        If `True`, raise a warning if a package cannot be imported.
1268
1269    install: bool, default True
1270        If `True`, attempt to install a missing package into the designated virtual environment.
1271        If `check_update` is True, install updates if available.
1272
1273    venv: Optional[str], default 'mrsm'
1274        The virtual environment in which to search for packages and to install packages into.
1275
1276    precheck: bool, default True
1277        If `True`, attempt to find module before importing (necessary for checking if modules exist
1278        and retaining lazy imports), otherwise assume lazy is `False`.
1279
1280    split: bool, default True
1281        If `True`, split packages' names on `'.'`.
1282
1283    check_update: bool, default False
1284        If `True` and `install` is `True`, install updates if the required minimum version
1285        does not match.
1286
1287    check_pypi: bool, default False
1288        If `True` and `check_update` is `True`, check PyPI when determining whether
1289        an update is required.
1290
1291    check_is_installed: bool, default True
1292        If `True`, check if the package is contained in the virtual environment.
1293
1294    allow_outside_venv: bool, default True
1295        If `True`, search outside of the specified virtual environment
1296        if the package cannot be found.
1297        Setting to `False` will reinstall the package into a virtual environment, even if it
1298        is installed outside.
1299
1300    color: bool, default True
1301        If `False`, do not print ANSI colors.
1302
1303    Returns
1304    -------
1305    The specified modules. If they're not available and `install` is `True`, it will first
1306    download them into a virtual environment and return the modules.
1307
1308    Examples
1309    --------
1310    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1311    >>> pandas = attempt_import('pandas')
1312
1313    """
1314
1315    import importlib.util
1316
1317    ### to prevent recursion, check if parent Meerschaum package is being imported
1318    if names == ('meerschaum',):
1319        return _import_module('meerschaum')
1320
1321    if venv == 'mrsm' and _import_hook_venv is not None:
1322        if debug:
1323            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1324        venv = _import_hook_venv
1325
1326    _warnings = _import_module('meerschaum.utils.warnings')
1327    warn_function = _warnings.warn
1328
1329    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1330        with Venv(venv=venv, debug=debug):
1331            ### determine the import method (lazy vs normal)
1332            from meerschaum.utils.misc import filter_keywords
1333            import_method = (
1334                _import_module if not lazy
1335                else lazy_import
1336            )
1337            try:
1338                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1339            except Exception as e:
1340                if warn:
1341                    import traceback
1342                    traceback.print_exception(type(e), e, e.__traceback__)
1343                    warn_function(
1344                        f"Failed to import module '{_name}'.\nException:\n{e}",
1345                        ImportWarning,
1346                        stacklevel = (5 if lazy else 4),
1347                        color = False,
1348                    )
1349                mod = None
1350        return mod
1351
1352    modules = []
1353    for name in names:
1354        ### Check if package is a declared dependency.
1355        root_name = name.split('.')[0] if split else name
1356        install_name = _import_to_install_name(root_name)
1357
1358        if install_name is None:
1359            install_name = root_name
1360            if warn and root_name != 'plugins':
1361                warn_function(
1362                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1363                    ImportWarning,
1364                    stacklevel = 3,
1365                    color = False
1366                )
1367
1368        ### Determine if the package exists.
1369        if precheck is False:
1370            found_module = (
1371                do_import(
1372                    name, debug=debug, warn=False, venv=venv, color=color,
1373                    check_update=False, check_pypi=False, split=split,
1374                ) is not None
1375            )
1376        else:
1377            if check_is_installed:
1378                with _locks['_is_installed_first_check']:
1379                    if not _is_installed_first_check.get(name, False):
1380                        package_is_installed = is_installed(
1381                            name,
1382                            venv = venv,
1383                            split = split,
1384                            allow_outside_venv = allow_outside_venv,
1385                            debug = debug,
1386                        )
1387                        _is_installed_first_check[name] = package_is_installed
1388                    else:
1389                        package_is_installed = _is_installed_first_check[name]
1390            else:
1391                package_is_installed = _is_installed_first_check.get(
1392                    name,
1393                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1394                )
1395            found_module = package_is_installed
1396
1397        if not found_module:
1398            if install:
1399                if not pip_install(
1400                    install_name,
1401                    venv = venv,
1402                    split = False,
1403                    check_update = check_update,
1404                    color = color,
1405                    debug = debug
1406                ) and warn:
1407                    warn_function(
1408                        f"Failed to install '{install_name}'.",
1409                        ImportWarning,
1410                        stacklevel = 3,
1411                        color = False,
1412                    )
1413            elif warn:
1414                ### Raise a warning if we can't find the package and install = False.
1415                warn_function(
1416                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1417                     + "some features will not work correctly."
1418                     + "\n\nSet install=True when calling attempt_import.\n"),
1419                    ImportWarning,
1420                    stacklevel = 3,
1421                    color = False,
1422                )
1423
1424        ### Do the import. Will be lazy if lazy=True.
1425        m = do_import(
1426            name, debug=debug, warn=warn, venv=venv, color=color,
1427            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1428        )
1429        modules.append(m)
1430
1431    modules = tuple(modules)
1432    if len(modules) == 1:
1433        return modules[0]
1434    return modules

Raise a warning if packages are not installed; otherwise import and return modules. If lazy is True, return lazy-imported modules.

Returns tuple of modules if multiple names are provided, else returns one module.

Parameters
  • names (List[str]): The packages to be imported.
  • lazy (bool, default True): If True, lazily load packages.
  • warn (bool, default True): If True, raise a warning if a package cannot be imported.
  • install (bool, default True): If True, attempt to install a missing package into the designated virtual environment. If check_update is True, install updates if available.
  • venv (Optional[str], default 'mrsm'): The virtual environment in which to search for packages and to install packages into.
  • precheck (bool, default True): If True, attempt to find module before importing (necessary for checking if modules exist and retaining lazy imports), otherwise assume lazy is False.
  • split (bool, default True): If True, split packages' names on '.'.
  • check_update (bool, default False): If True and install is True, install updates if the required minimum version does not match.
  • check_pypi (bool, default False): If True and check_update is True, check PyPI when determining whether an update is required.
  • check_is_installed (bool, default True): If True, check if the package is contained in the virtual environment.
  • allow_outside_venv (bool, default True): If True, search outside of the specified virtual environment if the package cannot be found. Setting to False will reinstall the package into a virtual environment, even if it is installed outside.
  • color (bool, default True): If False, do not print ANSI colors.
Returns
  • The specified modules. If they're not available and install is True, it will first
  • download them into a virtual environment and return the modules.
Examples
>>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
>>> pandas = attempt_import('pandas')
def lazy_import( name: str, local_name: str = None, **kw) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1437def lazy_import(
1438    name: str,
1439    local_name: str = None,
1440    **kw
1441) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1442    """
1443    Lazily import a package.
1444    """
1445    from meerschaum.utils.packages.lazy_loader import LazyLoader
1446    if local_name is None:
1447        local_name = name
1448    return LazyLoader(
1449        local_name,
1450        globals(),
1451        name,
1452        **kw
1453    )

Lazily import a package.

def pandas_name() -> str:
1456def pandas_name() -> str:
1457    """
1458    Return the configured name for `pandas`.
1459    
1460    Below are the expected possible values:
1461
1462    - 'pandas'
1463    - 'modin.pandas'
1464    - 'dask.dataframe'
1465
1466    """
1467    from meerschaum.config import get_config
1468    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1469    if pandas_module_name == 'modin':
1470        pandas_module_name = 'modin.pandas'
1471    elif pandas_module_name == 'dask':
1472        pandas_module_name = 'dask.dataframe'
1473
1474    return pandas_module_name

Return the configured name for pandas.

Below are the expected possible values:

  • 'pandas'
  • 'modin.pandas'
  • 'dask.dataframe'
emitted_pandas_warning: bool = False
def import_pandas(debug: bool = False, lazy: bool = False, **kw) -> "'ModuleType'":
1478def import_pandas(
1479    debug: bool = False,
1480    lazy: bool = False,
1481    **kw
1482) -> 'ModuleType':
1483    """
1484    Quality-of-life function to attempt to import the configured version of `pandas`.
1485    """
1486    pandas_module_name = pandas_name()
1487    global emitted_pandas_warning
1488
1489    if pandas_module_name != 'pandas':
1490        with _locks['emitted_pandas_warning']:
1491            if not emitted_pandas_warning:
1492                from meerschaum.utils.warnings import warn
1493                emitted_pandas_warning = True
1494                warn(
1495                    (
1496                        "You are using an alternative Pandas implementation "
1497                        + f"'{pandas_module_name}'"
1498                        + "\n   Features may not work as expected."
1499                    ),
1500                    stack=False,
1501                )
1502
1503    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1504    pandas, pyarrow = attempt_import('pandas', 'pyarrow', debug=debug, lazy=False, **kw)
1505    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1506    return pd

Quality-of-life function to attempt to import the configured version of pandas.

def import_rich(lazy: bool = True, debug: bool = False, **kw: Any) -> "'ModuleType'":
1509def import_rich(
1510    lazy: bool = True,
1511    debug: bool = False,
1512    **kw: Any
1513) -> 'ModuleType':
1514    """
1515    Quality of life function for importing `rich`.
1516    """
1517    from meerschaum.utils.formatting import ANSI, UNICODE
1518    ## need typing_extensions for `from rich import box`
1519    typing_extensions = attempt_import(
1520        'typing_extensions', lazy=False, debug=debug
1521    )
1522    pygments = attempt_import(
1523        'pygments', lazy=False,
1524    )
1525    rich = attempt_import(
1526        'rich', lazy=lazy,
1527        **kw
1528    )
1529    return rich

Quality of life function for importing rich.

def import_dcc(warn=False, **kw) -> "'ModuleType'":
1543def import_dcc(warn=False, **kw) -> 'ModuleType':
1544    """
1545    Import Dash Core Components (`dcc`).
1546    """
1547    return (
1548        attempt_import('dash_core_components', warn=warn, **kw)
1549        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1550    )

Import Dash Core Components (dcc).

def import_html(warn=False, **kw) -> "'ModuleType'":
1553def import_html(warn=False, **kw) -> 'ModuleType':
1554    """
1555    Import Dash HTML Components (`html`).
1556    """
1557    return (
1558        attempt_import('dash_html_components', warn=warn, **kw)
1559        if _dash_less_than_2(warn=warn, **kw)
1560        else attempt_import('dash.html', warn=warn, **kw)
1561    )

Import Dash HTML Components (html).

def get_modules_from_package( package: "'package'", names: bool = False, recursive: bool = False, lazy: bool = False, modules_venvs: bool = False, debug: bool = False):
1564def get_modules_from_package(
1565    package: 'package',
1566    names: bool = False,
1567    recursive: bool = False,
1568    lazy: bool = False,
1569    modules_venvs: bool = False,
1570    debug: bool = False
1571):
1572    """
1573    Find and import all modules in a package.
1574    
1575    Returns
1576    -------
1577    Either list of modules or tuple of lists.
1578    """
1579    from os.path import dirname, join, isfile, isdir, basename
1580    import glob
1581
1582    pattern = '*' if recursive else '*.py'
1583    package_path = dirname(package.__file__ or package.__path__[0])
1584    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1585    _all = [
1586        basename(f)[:-3] if isfile(f) else basename(f)
1587        for f in module_names
1588            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1589               and not f.endswith('__init__.py')
1590               and not f.endswith('__pycache__')
1591    ]
1592
1593    if debug:
1594        from meerschaum.utils.debug import dprint
1595        dprint(str(_all))
1596    modules = []
1597    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1598        ### there's probably a better way than a try: catch but it'll do for now
1599        try:
1600            ### if specified, activate the module's virtual environment before importing.
1601            ### NOTE: this only considers the filename, so two modules from different packages
1602            ### may end up sharing virtual environments.
1603            if modules_venvs:
1604                activate_venv(module_name.split('.')[-1], debug=debug)
1605            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1606            modules.append(m)
1607        except Exception as e:
1608            if debug:
1609                dprint(str(e))
1610        finally:
1611            if modules_venvs:
1612                deactivate_venv(module_name.split('.')[-1], debug=debug)
1613    if names:
1614        return _all, modules
1615
1616    return modules

Find and import all modules in a package.

Returns
  • Either list of modules or tuple of lists.
def import_children( package: "Optional['ModuleType']" = None, package_name: Optional[str] = None, types: Optional[List[str]] = None, lazy: bool = True, recursive: bool = False, debug: bool = False) -> "List['ModuleType']":
1619def import_children(
1620    package: Optional['ModuleType'] = None,
1621    package_name: Optional[str] = None,
1622    types : Optional[List[str]] = None,
1623    lazy: bool = True,
1624    recursive: bool = False,
1625    debug: bool = False
1626) -> List['ModuleType']:
1627    """
1628    Import all functions in a package to its `__init__`.
1629
1630    Parameters
1631    ----------
1632    package: Optional[ModuleType], default None
1633        Package to import its functions into.
1634        If `None` (default), use parent.
1635
1636    package_name: Optional[str], default None
1637        Name of package to import its functions into
1638        If None (default), use parent.
1639
1640    types: Optional[List[str]], default None
1641        Types of members to return.
1642        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1643
1644    Returns
1645    -------
1646    A list of modules.
1647    """
1648    import sys, inspect
1649
1650    if types is None:
1651        types = ['method', 'builtin', 'function', 'class', 'module']
1652
1653    ### if package_name and package are None, use parent
1654    if package is None and package_name is None:
1655        package_name = inspect.stack()[1][0].f_globals['__name__']
1656
1657    ### populate package or package_name from other other
1658    if package is None:
1659        package = sys.modules[package_name]
1660    elif package_name is None:
1661        package_name = package.__name__
1662
1663    ### Set attributes in sys module version of package.
1664    ### Kinda like setting a dictionary
1665    ###   functions[name] = func
1666    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1667    _all, members = [], []
1668    objects = []
1669    for module in modules:
1670        _objects = []
1671        for ob in inspect.getmembers(module):
1672            for t in types:
1673                ### ob is a tuple of (name, object)
1674                if getattr(inspect, 'is' + t)(ob[1]):
1675                    _objects.append(ob)
1676
1677        if 'module' in types:
1678            _objects.append((module.__name__.split('.')[0], module))
1679        objects += _objects
1680    for ob in objects:
1681        setattr(sys.modules[package_name], ob[0], ob[1])
1682        _all.append(ob[0])
1683        members.append(ob[1])
1684
1685    if debug:
1686        from meerschaum.utils.debug import dprint
1687        dprint(str(_all))
1688    ### set __all__ for import *
1689    setattr(sys.modules[package_name], '__all__', _all)
1690    return members

Import all functions in a package to its __init__.

Parameters
  • package (Optional[ModuleType], default None): Package to import its functions into. If None (default), use parent.
  • package_name (Optional[str], default None): Name of package to import its functions into If None (default), use parent.
  • types (Optional[List[str]], default None): Types of members to return. Defaults are ['method', 'builtin', 'class', 'function', 'package', 'module']
Returns
  • A list of modules.
def reload_package( package: str, skip_submodules: Optional[List[str]] = None, lazy: bool = False, debug: bool = False, **kw: Any):
1694def reload_package(
1695    package: str,
1696    skip_submodules: Optional[List[str]] = None,
1697    lazy: bool = False,
1698    debug: bool = False,
1699    **kw: Any
1700):
1701    """
1702    Recursively load a package's subpackages, even if they were not previously loaded.
1703    """
1704    import sys
1705    if isinstance(package, str):
1706        package_name = package
1707    else:
1708        try:
1709            package_name = package.__name__
1710        except Exception as e:
1711            package_name = str(package)
1712
1713    skip_submodules = skip_submodules or []
1714    if 'meerschaum.utils.packages' not in skip_submodules:
1715        skip_submodules.append('meerschaum.utils.packages')
1716    def safeimport():
1717        subs = [
1718            m for m in sys.modules
1719            if m.startswith(package_name + '.')
1720        ]
1721        subs_to_skip = []
1722        for skip_mod in skip_submodules:
1723            for mod in subs:
1724                if mod.startswith(skip_mod):
1725                    subs_to_skip.append(mod)
1726                    continue
1727
1728        subs = [m for m in subs if m not in subs_to_skip]
1729        for module_name in subs:
1730            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1731        if not subs_to_skip:
1732            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1733
1734        return _import_module(package_name)
1735
1736    return safeimport()

Recursively load a package's subpackages, even if they were not previously loaded.

def reload_meerschaum(debug: bool = False) -> Tuple[bool, str]:
1739def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1740    """
1741    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1742    """
1743    reload_package(
1744        'meerschaum',
1745        skip_submodules = [
1746            'meerschaum._internal.shell',
1747            'meerschaum.utils.pool',
1748        ]
1749    )
1750
1751    from meerschaum.plugins import reload_plugins
1752    from meerschaum._internal.shell.Shell import _insert_shell_actions
1753    reload_plugins(debug=debug)
1754    _insert_shell_actions()
1755    return True, "Success"

Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.

def is_installed( import_name: str, venv: Optional[str] = 'mrsm', split: bool = True, allow_outside_venv: bool = True, debug: bool = False) -> bool:
1758def is_installed(
1759    import_name: str,
1760    venv: Optional[str] = 'mrsm',
1761    split: bool = True,
1762    allow_outside_venv: bool = True,
1763    debug: bool = False,
1764) -> bool:
1765    """
1766    Check whether a package is installed.
1767
1768    Parameters
1769    ----------
1770    import_name: str
1771        The import name of the module.
1772
1773    venv: Optional[str], default 'mrsm'
1774        The venv in which to search for the module.
1775
1776    split: bool, default True
1777        If `True`, split on periods to determine the root module name.
1778
1779    allow_outside_venv: bool, default True
1780        If `True`, search outside of the specified virtual environment
1781        if the package cannot be found.
1782
1783    Returns
1784    -------
1785    A bool indicating whether a package may be imported.
1786    """
1787    if debug:
1788        from meerschaum.utils.debug import dprint
1789    root_name = import_name.split('.')[0] if split else import_name
1790    import importlib.util
1791    with Venv(venv, debug=debug):
1792        try:
1793            spec_path = pathlib.Path(
1794                get_module_path(root_name, venv=venv, debug=debug)
1795                or
1796                (
1797                    importlib.util.find_spec(root_name).origin 
1798                    if venv is not None and allow_outside_venv
1799                    else None
1800                )
1801            )
1802        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1803            spec_path = None
1804
1805        found = (
1806            not need_update(
1807                None,
1808                import_name=root_name,
1809                _run_determine_version=False,
1810                check_pypi=False,
1811                version=determine_version(
1812                    spec_path,
1813                    venv=venv,
1814                    debug=debug,
1815                    import_name=root_name,
1816                ),
1817                debug=debug,
1818            )
1819        ) if spec_path is not None else False
1820
1821    return found

Check whether a package is installed.

Parameters
  • import_name (str): The import name of the module.
  • venv (Optional[str], default 'mrsm'): The venv in which to search for the module.
  • split (bool, default True): If True, split on periods to determine the root module name.
  • allow_outside_venv (bool, default True): If True, search outside of the specified virtual environment if the package cannot be found.
Returns
  • A bool indicating whether a package may be imported.
def venv_contains_package( import_name: str, venv: Optional[str] = 'mrsm', split: bool = True, debug: bool = False) -> bool:
1824def venv_contains_package(
1825    import_name: str,
1826    venv: Optional[str] = 'mrsm',
1827    split: bool = True,
1828    debug: bool = False,
1829) -> bool:
1830    """
1831    Search the contents of a virtual environment for a package.
1832    """
1833    import site
1834    import pathlib
1835    root_name = import_name.split('.')[0] if split else import_name
1836    return get_module_path(root_name, venv=venv, debug=debug) is not None

Search the contents of a virtual environment for a package.

def package_venv(package: "'ModuleType'") -> Optional[str]:
1839def package_venv(package: 'ModuleType') -> Union[str, None]:
1840    """
1841    Inspect a package and return the virtual environment in which it presides.
1842    """
1843    import os
1844    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1845    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1846        return None
1847    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]

Inspect a package and return the virtual environment in which it presides.

def ensure_readline() -> "'ModuleType'":
1850def ensure_readline() -> 'ModuleType':
1851    """Make sure that the `readline` package is able to be imported."""
1852    import sys
1853    try:
1854        import readline
1855    except ImportError:
1856        readline = None
1857
1858    if readline is None:
1859        import platform
1860        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1861        try:
1862            rl = attempt_import(
1863                rl_name,
1864                lazy=False,
1865                install=True,
1866                venv=None,
1867                warn=False,
1868            )
1869        except (ImportError, ModuleNotFoundError):
1870            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1871                print(f"Unable to import {rl_name}!", file=sys.stderr)
1872                sys.exit(1)
1873
1874    sys.modules['readline'] = readline
1875    return readline

Make sure that the readline package is able to be imported.

def use_uv() -> bool:
1899def use_uv() -> bool:
1900    """
1901    Return whether `uv` is available and enabled.
1902    """
1903    from meerschaum.utils.misc import is_android
1904    if is_android():
1905        return False
1906
1907    if not is_uv_enabled():
1908        return False
1909
1910    try:
1911        import uv
1912        uv_bin = uv.find_uv_bin()
1913    except (ImportError, FileNotFoundError):
1914        uv_bin = None
1915
1916    if uv_bin is None:
1917        return False
1918
1919    return True

Return whether uv is available and enabled.

def is_uv_enabled() -> bool:
1922def is_uv_enabled() -> bool:
1923    """
1924    Return whether the user has disabled `uv`.
1925    """
1926    from meerschaum.utils.misc import is_android
1927    if is_android():
1928        return False
1929
1930    from meerschaum.utils.venv import inside_venv
1931
1932    if inside_venv():
1933        return False
1934
1935    try:
1936        import yaml
1937    except ImportError:
1938        return False
1939
1940    from meerschaum.config import get_config
1941    enabled = get_config('system', 'experimental', 'uv_pip')
1942    return enabled

Return whether the user has disabled uv.