meerschaum.utils.packages

Functions for managing packages and virtual environments reside here.

   1#! /usr/bin/env python
   2# -*- coding: utf-8 -*-
   3# vim:fenc=utf-8
   4
   5"""
   6Functions for managing packages and virtual environments reside here.
   7"""
   8
   9from __future__ import annotations
  10
  11import importlib.util, os, pathlib
  12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
  13from meerschaum.utils.threading import Lock, RLock
  14from meerschaum.utils.packages._packages import packages, all_packages, get_install_names
  15from meerschaum.utils.venv import (
  16    activate_venv,
  17    deactivate_venv,
  18    venv_executable,
  19    venv_exec,
  20    venv_exists,
  21    venv_target_path,
  22    inside_venv,
  23    Venv,
  24    init_venv,
  25)
  26
  27_import_module = importlib.import_module
  28_import_hook_venv = None
  29_locks = {
  30    '_pkg_resources_get_distribution': RLock(),
  31    'import_versions': RLock(),
  32    '_checked_for_updates': RLock(),
  33    '_is_installed_first_check': RLock(),
  34    'emitted_pandas_warning': RLock(),
  35}
  36_checked_for_updates = set()
  37_is_installed_first_check: Dict[str, bool] = {}
  38
  39def get_module_path(
  40        import_name: str,
  41        venv: Optional[str] = 'mrsm',
  42        debug: bool = False,
  43        _try_install_name_on_fail: bool = True,
  44    ) -> Union[pathlib.Path, None]:
  45    """
  46    Get a module's path without importing.
  47    """
  48    if debug:
  49        from meerschaum.utils.debug import dprint
  50    if not _try_install_name_on_fail:
  51        install_name = _import_to_install_name(import_name, with_version=False)
  52        install_name_lower = install_name.lower().replace('-', '_')
  53        import_name_lower = install_name_lower
  54    else:
  55        import_name_lower = import_name.lower().replace('-', '_')
  56    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
  57    if not vtp.exists():
  58        if debug:
  59            dprint(f"Venv '{venv}' does not exist, cannot import '{import_name}'.", color=False)
  60        return None
  61    candidates = []
  62    for file_name in os.listdir(vtp):
  63        file_name_lower = file_name.lower().replace('-', '_')
  64        if not file_name_lower.startswith(import_name_lower):
  65            continue
  66        if file_name.endswith('dist_info'):
  67            continue
  68        file_path = vtp / file_name
  69
  70        ### Most likely: Is a directory with __init__.py
  71        if file_name_lower == import_name_lower and file_path.is_dir():
  72            init_path = file_path / '__init__.py'
  73            if init_path.exists():
  74                candidates.append(init_path)
  75
  76        ### May be a standalone .py file.
  77        elif file_name_lower == import_name_lower + '.py':
  78            candidates.append(file_path)
  79
  80        ### Compiled wheels (e.g. pyodbc)
  81        elif file_name_lower.startswith(import_name_lower + '.'):
  82            candidates.append(file_path)
  83
  84    if len(candidates) == 1:
  85        return candidates[0]
  86
  87    if not candidates:
  88        if _try_install_name_on_fail:
  89            return get_module_path(
  90                import_name, venv=venv, debug=debug,
  91                _try_install_name_on_fail=False
  92            )
  93        return None
  94
  95    specs_paths = []
  96    for candidate_path in candidates:
  97        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
  98        if spec is not None:
  99            return candidate_path
 100    
 101    return None
 102
 103
 104def manually_import_module(
 105        import_name: str,
 106        venv: Optional[str] = 'mrsm',
 107        check_update: bool = True,
 108        check_pypi: bool = False,
 109        install: bool = True,
 110        split: bool = True,
 111        warn: bool = True,
 112        color: bool = True,
 113        debug: bool = False,
 114        use_sys_modules: bool = True,
 115    ) -> Union['ModuleType', None]:
 116    """
 117    Manually import a module from a virtual environment (or the base environment).
 118
 119    Parameters
 120    ----------
 121    import_name: str
 122        The name of the module.
 123        
 124    venv: Optional[str], default 'mrsm'
 125        The virtual environment to read from.
 126
 127    check_update: bool, default True
 128        If `True`, examine whether the available version of the package meets the required version.
 129
 130    check_pypi: bool, default False
 131        If `True`, check PyPI for updates before importing.
 132
 133    install: bool, default True
 134        If `True`, install the package if it's not installed or needs an update.
 135
 136    split: bool, default True
 137        If `True`, split `import_name` on periods to get the package name.
 138
 139    warn: bool, default True
 140        If `True`, raise a warning if the package cannot be imported.
 141
 142    color: bool, default True
 143        If `True`, use color output for debug and warning text.
 144
 145    debug: bool, default False
 146        Verbosity toggle.
 147
 148    use_sys_modules: bool, default True
 149        If `True`, return the module in `sys.modules` if it exists.
 150        Otherwise continue with manually importing.
 151
 152    Returns
 153    -------
 154    The specified module or `None` if it can't be imported.
 155
 156    """
 157    import sys
 158    _previously_imported = import_name in sys.modules
 159    if _previously_imported and use_sys_modules:
 160        return sys.modules[import_name]
 161    if debug:
 162        from meerschaum.utils.debug import dprint
 163    from meerschaum.utils.warnings import warn as warn_function
 164    import warnings
 165    root_name = import_name.split('.')[0] if split else import_name
 166    install_name = _import_to_install_name(root_name)
 167
 168    root_path = get_module_path(root_name, venv=venv)
 169    if root_path is None:
 170        return None
 171    mod_path = root_path
 172    if mod_path.is_dir():
 173        for _dir in import_name.split('.')[:-1]:
 174            mod_path = mod_path / _dir
 175            possible_end_module_filename = import_name.split('.')[-1] + '.py'
 176            try:
 177                mod_path = (
 178                    (mod_path / possible_end_module_filename)
 179                    if possible_end_module_filename in os.listdir(mod_path)
 180                    else (
 181                        mod_path / import_name.split('.')[-1] / '__init__.py'
 182                    )
 183                )
 184            except Exception as e:
 185                mod_path = None
 186
 187    spec = (
 188        importlib.util.find_spec(import_name) if mod_path is None or not mod_path.exists()
 189        else importlib.util.spec_from_file_location(import_name, str(mod_path))
 190    )
 191    root_spec = (
 192        importlib.util.find_spec(root_name) if not root_path.exists()
 193        else importlib.util.spec_from_file_location(root_name, str(root_path))
 194    )
 195
 196    ### Check for updates before importing.
 197    _version = (
 198        determine_version(
 199            pathlib.Path(root_spec.origin),
 200            import_name=root_name, venv=venv, debug=debug
 201        ) if root_spec is not None and root_spec.origin is not None else None
 202    )
 203
 204    if _version is not None:
 205        if check_update:
 206            if need_update(
 207                None,
 208                import_name = root_name,
 209                version = _version,
 210                check_pypi = check_pypi,
 211                debug = debug,
 212            ):
 213                if install:
 214                    if not pip_install(
 215                        root_name,
 216                        venv = venv,
 217                        split = False,
 218                        check_update = check_update,
 219                        color = color,
 220                        debug = debug
 221                    ) and warn:
 222                        warn_function(
 223                            f"There's an update available for '{install_name}', "
 224                            + "but it failed to install. "
 225                            + "Try installig via Meerschaum with "
 226                            + "`install packages '{install_name}'`.",
 227                            ImportWarning,
 228                            stacklevel = 3,
 229                            color = False,
 230                        )
 231                elif warn:
 232                    warn_function(
 233                        f"There's an update available for '{root_name}'.",
 234                        stack = False,
 235                        color = False,
 236                    )
 237                spec = (
 238                    importlib.util.find_spec(import_name)
 239                    if mod_path is None or not mod_path.exists()
 240                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
 241                )
 242
 243
 244    if spec is None:
 245        try:
 246            mod = _import_module(import_name)
 247        except Exception as e:
 248            mod = None
 249        return mod
 250
 251    with Venv(venv, debug=debug):
 252        mod = importlib.util.module_from_spec(spec)
 253        old_sys_mod = sys.modules.get(import_name, None)
 254        sys.modules[import_name] = mod
 255
 256        try:
 257            with warnings.catch_warnings():
 258                warnings.filterwarnings('ignore', 'The NumPy')
 259                spec.loader.exec_module(mod)
 260        except Exception as e:
 261            pass
 262        mod = _import_module(import_name)
 263        if old_sys_mod is not None:
 264            sys.modules[import_name] = old_sys_mod
 265        else:
 266            del sys.modules[import_name]
 267    
 268    return mod
 269
 270
 271def _import_to_install_name(import_name: str, with_version: bool = True) -> str:
 272    """
 273    Try to translate an import name to an installation name.
 274    """
 275    install_name = all_packages.get(import_name, import_name)
 276    if with_version:
 277        return install_name
 278    return get_install_no_version(install_name)
 279
 280
 281def _import_to_dir_name(import_name: str) -> str:
 282    """
 283    Translate an import name to the package name in the sites-packages directory.
 284    """
 285    import re
 286    return re.split(
 287        r'[<>=\[]', all_packages.get(import_name, import_name)
 288    )[0].replace('-', '_').lower() 
 289
 290
 291def _install_to_import_name(install_name: str) -> str:
 292    """
 293    Translate an installation name to a package's import name.
 294    """
 295    _install_no_version = get_install_no_version(install_name)
 296    return get_install_names().get(_install_no_version, _install_no_version)
 297
 298
 299def get_install_no_version(install_name: str) -> str:
 300    """
 301    Strip the version information from the install name.
 302    """
 303    import re
 304    return re.split(r'[\[=<>,! \]]', install_name)[0]
 305
 306
 307import_versions = {}
 308def determine_version(
 309        path: pathlib.Path,
 310        import_name: Optional[str] = None,
 311        venv: Optional[str] = 'mrsm',
 312        search_for_metadata: bool = True,
 313        split: bool = True,
 314        warn: bool = False,
 315        debug: bool = False,
 316    ) -> Union[str, None]:
 317    """
 318    Determine a module's `__version__` string from its filepath.
 319    
 320    First it searches for pip metadata, then it attempts to import the module in a subprocess.
 321
 322    Parameters
 323    ----------
 324    path: pathlib.Path
 325        The file path of the module.
 326
 327    import_name: Optional[str], default None
 328        The name of the module. If omitted, it will be determined from the file path.
 329        Defaults to `None`.
 330
 331    venv: Optional[str], default 'mrsm'
 332        The virtual environment of the Python interpreter to use if importing is necessary.
 333
 334    search_for_metadata: bool, default True
 335        If `True`, search the pip site_packages directory (assumed to be the parent)
 336        for the corresponding dist-info directory.
 337
 338    warn: bool, default True
 339        If `True`, raise a warning if the module fails to import in the subprocess.
 340
 341    split: bool, default True
 342        If `True`, split the determined import name by periods to get the room name.
 343
 344    Returns
 345    -------
 346    The package's version string if available or `None`.
 347    If multiple versions are found, it will trigger an import in a subprocess.
 348
 349    """
 350    with _locks['import_versions']:
 351        if venv not in import_versions:
 352            import_versions[venv] = {}
 353    import importlib.metadata
 354    import re, os
 355    old_cwd = os.getcwd()
 356    if debug:
 357        from meerschaum.utils.debug import dprint
 358    from meerschaum.utils.warnings import warn as warn_function
 359    if import_name is None:
 360        import_name = path.parent.stem if path.stem == '__init__' else path.stem
 361        import_name = import_name.split('.')[0] if split else import_name
 362    if import_name in import_versions[venv]:
 363        return import_versions[venv][import_name]
 364    _version = None
 365    module_parent_dir = (
 366        path.parent.parent if path.stem == '__init__' else path.parent
 367    ) if path is not None else venv_target_path(venv, debug=debug)
 368
 369    installed_dir_name = _import_to_dir_name(import_name)
 370    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
 371
 372    ### First, check if a dist-info directory exists.
 373    _found_versions = []
 374    if search_for_metadata:
 375        for filename in os.listdir(module_parent_dir):
 376            if not filename.endswith('.dist-info'):
 377                continue
 378            filename_lower = filename.lower()
 379            if not filename_lower.startswith(clean_installed_dir_name + '-'):
 380                continue
 381            _v = filename.replace('.dist-info', '').split("-")[-1]
 382            _found_versions.append(_v)
 383
 384    if len(_found_versions) == 1:
 385        _version = _found_versions[0]
 386        with _locks['import_versions']:
 387            import_versions[venv][import_name] = _version
 388        return _found_versions[0]
 389
 390    if not _found_versions:
 391        try:
 392            import importlib.metadata as importlib_metadata
 393        except ImportError:
 394            importlib_metadata = attempt_import(
 395                'importlib_metadata',
 396                debug=debug, check_update=False, precheck=False,
 397                color=False, check_is_installed=False, lazy=False,
 398            )
 399        try:
 400            os.chdir(module_parent_dir)
 401            _version = importlib_metadata.metadata(import_name)['Version']
 402        except Exception as e:
 403            _version = None
 404        finally:
 405            os.chdir(old_cwd)
 406
 407        if _version is not None:
 408            with _locks['import_versions']:
 409                import_versions[venv][import_name] = _version
 410            return _version
 411
 412    if debug:
 413        print(f'Found multiple versions for {import_name}: {_found_versions}')
 414
 415    module_parent_dir_str = module_parent_dir.as_posix()
 416
 417    ### Not a pip package, so let's try importing the module directly (in a subprocess).
 418    _no_version_str = 'no-version'
 419    code = (
 420        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
 421        + f"module = importlib.import_module('{import_name}');\n"
 422        + "try:\n"
 423        + "  print(module.__version__ , end='')\n"
 424        + "except:\n"
 425        + f"  print('{_no_version_str}', end='')"
 426    )
 427    exit_code, stdout_bytes, stderr_bytes = venv_exec(
 428        code, venv=venv, with_extras=True, debug=debug
 429    )
 430    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
 431    _version = stdout.split('\n')[-1] if exit_code == 0 else None
 432    _version = _version if _version != _no_version_str else None
 433
 434    if _version is None:
 435        _version = _get_package_metadata(import_name, venv).get('version', None)
 436    if _version is None and warn:
 437        warn_function(
 438            f"Failed to determine a version for '{import_name}':\n{stderr}",
 439            stack = False
 440        )
 441
 442    ### If `__version__` doesn't exist, return `None`.
 443    import_versions[venv][import_name] = _version
 444    return _version
 445
 446
 447def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]:
 448    """
 449    Get a package's metadata from pip.
 450    This is useful for getting a version when no `__version__` is defined
 451    and multiple versions are installed.
 452
 453    Parameters
 454    ----------
 455    import_name: str
 456        The package's import or installation name.
 457
 458    venv: Optional[str]
 459        The virtual environment which contains the package.
 460
 461    Returns
 462    -------
 463    A dictionary of metadata from pip.
 464    """
 465    import re
 466    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 467    install_name = _import_to_install_name(import_name)
 468    _args = ['show', install_name]
 469    if venv is not None:
 470        cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 471        _args += ['--cache-dir', str(cache_dir_path)]
 472    proc = run_python_package(
 473        'pip', _args,
 474        capture_output=True, as_proc=True, venv=venv, universal_newlines=True,
 475    )
 476    outs, errs = proc.communicate()
 477    lines = outs.split('\n')
 478    meta = {}
 479    for line in lines:
 480        vals = line.split(": ")
 481        if len(vals) != 2:
 482            continue
 483        k, v = vals[0].lower(), vals[1]
 484        if v and 'UNKNOWN' not in v:
 485            meta[k] = v
 486    return meta
 487
 488
 489def need_update(
 490        package: Optional['ModuleType'] = None,
 491        install_name: Optional[str] = None,
 492        import_name: Optional[str] = None,
 493        version: Optional[str] = None,
 494        check_pypi: bool = False,
 495        split: bool = True,
 496        color: bool = True,
 497        debug: bool = False,
 498        _run_determine_version: bool = True,
 499    ) -> bool:
 500    """
 501    Check if a Meerschaum dependency needs an update.
 502    Returns a bool for whether or not a package needs to be updated.
 503
 504    Parameters
 505    ----------
 506    package: 'ModuleType'
 507        The module of the package to be updated.
 508
 509    install_name: Optional[str], default None
 510        If provided, use this string to determine the required version.
 511        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
 512
 513    import_name:
 514        If provided, override the package's `__name__` string.
 515
 516    version: Optional[str], default None
 517        If specified, override the package's `__version__` string.
 518
 519    check_pypi: bool, default False
 520        If `True`, check pypi.org for updates.
 521        Defaults to `False`.
 522
 523    split: bool, default True
 524        If `True`, split the module's name on periods to detrive the root name.
 525        Defaults to `True`.
 526
 527    color: bool, default True
 528        If `True`, format debug output.
 529        Defaults to `True`.
 530
 531    debug: bool, default True
 532        Verbosity toggle.
 533
 534    Returns
 535    -------
 536    A bool indicating whether the package requires an update.
 537
 538    """
 539    if debug:
 540        from meerschaum.utils.debug import dprint
 541    from meerschaum.utils.warnings import warn as warn_function
 542    import re
 543    root_name = (
 544        package.__name__.split('.')[0] if split else package.__name__
 545    ) if import_name is None else (
 546        import_name.split('.')[0] if split else import_name
 547    )
 548    install_name = install_name or _import_to_install_name(root_name)
 549    with _locks['_checked_for_updates']:
 550        if install_name in _checked_for_updates:
 551            return False
 552        _checked_for_updates.add(install_name)
 553
 554    _install_no_version = get_install_no_version(install_name)
 555    required_version = install_name.replace(_install_no_version, '')
 556    if ']' in required_version:
 557        required_version = required_version.split(']')[1]
 558
 559    ### No minimum version was specified, and we're not going to check PyPI.
 560    if not required_version and not check_pypi:
 561        return False
 562
 563    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
 564    if '.dev' in required_version:
 565        required_version = required_version.split('.dev')[0]
 566    if version and '.dev' in version:
 567        version = version.split('.dev')[0]
 568
 569    try:
 570        if not version:
 571            if not _run_determine_version:
 572                version = determine_version(
 573                    pathlib.Path(package.__file__),
 574                    import_name=root_name, warn=False, debug=debug
 575                )
 576        if version is None:
 577            return False
 578    except Exception as e:
 579        if debug:
 580            dprint(str(e), color=color)
 581            dprint("No version could be determined from the installed package.", color=color)
 582        return False
 583    split_version = version.split('.')
 584    last_part = split_version[-1]
 585    if len(split_version) == 2:
 586        version = '.'.join(split_version) + '.0'
 587    elif 'dev' in last_part or 'rc' in last_part:
 588        tag = 'dev' if 'dev' in last_part else 'rc'
 589        last_sep = '-'
 590        if not last_part.startswith(tag):
 591            last_part = f'-{tag}'.join(last_part.split(tag))
 592            last_sep = '.'
 593        version = '.'.join(split_version[:-1]) + last_sep + last_part
 594    elif len(split_version) > 3:
 595        version = '.'.join(split_version[:3])
 596
 597    packaging_version = attempt_import(
 598        'packaging.version', check_update=False, lazy=False, debug=debug,
 599    )
 600
 601    ### Get semver if necessary
 602    if required_version:
 603        semver_path = get_module_path('semver', debug=debug)
 604        if semver_path is None:
 605            pip_install(_import_to_install_name('semver'), debug=debug)
 606        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
 607    if check_pypi:
 608        ### Check PyPI for updates
 609        update_checker = attempt_import(
 610            'update_checker', lazy=False, check_update=False, debug=debug
 611        )
 612        checker = update_checker.UpdateChecker()
 613        result = checker.check(_install_no_version, version)
 614    else:
 615        ### Skip PyPI and assume we can't be sure.
 616        result = None
 617
 618    ### Compare PyPI's version with our own.
 619    if result is not None:
 620        ### We have a result from PyPI and a stated required version.
 621        if required_version:
 622            try:
 623                return semver.Version.parse(result.available_version).match(required_version)
 624            except AttributeError as e:
 625                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 626                semver = manually_import_module('semver', venv='mrsm')
 627                return semver.Version.parse(version).match(required_version)
 628            except Exception as e:
 629                if debug:
 630                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
 631                return False
 632
 633        ### If `check_pypi` and we don't have a required version, check if PyPI's version
 634        ### is newer than the installed version.
 635        else:
 636            return (
 637                packaging_version.parse(result.available_version) > 
 638                packaging_version.parse(version)
 639            )
 640
 641    ### We might be depending on a prerelease.
 642    ### Sanity check that the required version is not greater than the installed version. 
 643    try:
 644        return (
 645            (not semver.Version.parse(version).match(required_version))
 646            if required_version else False
 647        )
 648    except AttributeError as e:
 649        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
 650        semver = manually_import_module('semver', venv='mrsm', debug=debug)
 651        return (
 652            (not semver.Version.parse(version).match(required_version))
 653            if required_version else False
 654        )
 655    except Exception as e:
 656        print(f"Unable to parse version ({version}) for package '{import_name}'.")
 657        print(e)
 658        if debug:
 659            dprint(e)
 660        return False
 661    try:
 662        return (
 663            packaging_version.parse(version) > 
 664            packaging_version.parse(required_version)
 665        )
 666    except Exception as e:
 667        if debug:
 668            dprint(e)
 669        return False
 670    return False
 671
 672
 673def get_pip(venv: Optional[str] = 'mrsm', debug: bool=False) -> bool:
 674    """
 675    Download and run the get-pip.py script.
 676
 677    Parameters
 678    ----------
 679    debug: bool, default False
 680        Verbosity toggle.
 681
 682    Returns
 683    -------
 684    A bool indicating success.
 685
 686    """
 687    import sys, subprocess
 688    from meerschaum.utils.misc import wget
 689    from meerschaum.config._paths import CACHE_RESOURCES_PATH
 690    from meerschaum.config.static import STATIC_CONFIG
 691    url = STATIC_CONFIG['system']['urls']['get-pip.py']
 692    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
 693    try:
 694        wget(url, dest, color=False, debug=debug)
 695    except Exception as e:
 696        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
 697        sys.exit(1)
 698    if venv is not None:
 699        init_venv(venv=venv, debug=debug)
 700    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
 701    return subprocess.call(cmd_list, env=_get_pip_os_env()) == 0
 702
 703
 704def pip_install(
 705        *install_names: str,
 706        args: Optional[List[str]] = None,
 707        requirements_file_path: Union[pathlib.Path, str, None] = None,
 708        venv: Optional[str] = 'mrsm',
 709        split: bool = False,
 710        check_update: bool = True,
 711        check_pypi: bool = True,
 712        check_wheel: bool = True,
 713        _uninstall: bool = False,
 714        color: bool = True,
 715        silent: bool = False,
 716        debug: bool = False,
 717    ) -> bool:
 718    """
 719    Install packages from PyPI with `pip`.
 720
 721    Parameters
 722    ----------
 723    *install_names: str
 724        The installation names of packages to be installed.
 725        This includes version restrictions.
 726        Use `_import_to_install_name()` to get the predefined `install_name` for a package
 727        from its import name.
 728        
 729    args: Optional[List[str]], default None
 730        A list of command line arguments to pass to `pip`.
 731        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
 732
 733    requirements_file_path: Optional[pathlib.Path, str], default None
 734        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
 735
 736    venv: str, default 'mrsm'
 737        The virtual environment to install into.
 738
 739    split: bool, default False
 740        If `True`, split on periods and only install the root package name.
 741
 742    check_update: bool, default True
 743        If `True`, check if the package requires an update.
 744
 745    check_pypi: bool, default True
 746        If `True` and `check_update` is `True`, check PyPI for the latest version.
 747
 748    check_wheel: bool, default True
 749        If `True`, check if `wheel` is available.
 750
 751    _uninstall: bool, default False
 752        If `True`, uninstall packages instead.
 753
 754    color: bool, default True
 755        If `True`, include color in debug text.
 756
 757    silent: bool, default False
 758        If `True`, skip printing messages.
 759
 760    debug: bool, default False
 761        Verbosity toggle.
 762
 763    Returns
 764    -------
 765    A bool indicating success.
 766
 767    """
 768    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
 769    from meerschaum.utils.warnings import warn
 770    if args is None:
 771        args = ['--upgrade'] if not _uninstall else []
 772    if color:
 773        ANSI, UNICODE = True, True
 774    else:
 775        ANSI, UNICODE = False, False
 776    if check_wheel:
 777        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 778
 779    _args = list(args)
 780    have_pip = venv_contains_package('pip', venv=venv, debug=debug)
 781    import sys
 782    if not have_pip:
 783        if not get_pip(venv=venv, debug=debug):
 784            import sys
 785            minor = sys.version_info.minor
 786            print(
 787                "\nFailed to import `pip` and `ensurepip`.\n"
 788                + "If you are running Ubuntu/Debian, "
 789                + "you might need to install `python3.{minor}-distutils`:\n\n"
 790                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
 791                + "Please install pip and restart Meerschaum.\n\n"
 792                + "You can find instructions on installing `pip` here:\n"
 793                + "https://pip.pypa.io/en/stable/installing/"
 794            )
 795            sys.exit(1)
 796    
 797    with Venv(venv, debug=debug):
 798        if venv is not None:
 799            if '--ignore-installed' not in args and '-I' not in _args and not _uninstall:
 800                _args += ['--ignore-installed']
 801            if '--cache-dir' not in args and not _uninstall:
 802                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
 803                _args += ['--cache-dir', str(cache_dir_path)]
 804
 805        if 'pip' not in ' '.join(_args):
 806            if check_update and not _uninstall:
 807                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
 808                if need_update(pip, check_pypi=check_pypi, debug=debug):
 809                    _args.append(all_packages['pip'])
 810        
 811        _args = (['install'] if not _uninstall else ['uninstall']) + _args
 812
 813        if check_wheel and not _uninstall:
 814            if not have_wheel:
 815                if not pip_install(
 816                    'setuptools', 'wheel',
 817                    venv = venv,
 818                    check_update = False, check_pypi = False,
 819                    check_wheel = False, debug = debug,
 820                ):
 821                    warn(
 822                        f"Failed to install `setuptools` and `wheel` for virtual environment '{venv}'.",
 823                        color=False,
 824                    )
 825
 826        if requirements_file_path is not None:
 827            _args.append('-r')
 828            _args.append(str(pathlib.Path(requirements_file_path).resolve()))
 829
 830        if not ANSI and '--no-color' not in _args:
 831            _args.append('--no-color')
 832
 833        if '--no-input' not in _args:
 834            _args.append('--no-input')
 835
 836        if _uninstall and '-y' not in _args:
 837            _args.append('-y')
 838
 839        if '--no-warn-conflicts' not in _args and not _uninstall:
 840            _args.append('--no-warn-conflicts')
 841
 842        if '--disable-pip-version-check' not in _args:
 843            _args.append('--disable-pip-version-check')
 844
 845        if '--target' not in _args and '-t' not in _args and not _uninstall:
 846            if venv is not None:
 847                _args += ['--target', venv_target_path(venv, debug=debug)]
 848        elif (
 849            '--target' not in _args
 850                and '-t' not in _args
 851                and not inside_venv()
 852                and not _uninstall
 853        ):
 854            _args += ['--user']
 855
 856        if debug:
 857            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
 858                pass
 859        else:
 860            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
 861                pass
 862
 863        _packages = [
 864            (install_name if not _uninstall else get_install_no_version(install_name))
 865            for install_name in install_names
 866        ]
 867        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
 868        for p in _packages:
 869            msg += f'\n  - {p}'
 870        if not silent:
 871            print(msg)
 872
 873        if not _uninstall:
 874            for install_name in _packages:
 875                _install_no_version = get_install_no_version(install_name)
 876                if _install_no_version in ('pip', 'wheel'):
 877                    continue
 878                if not completely_uninstall_package(
 879                    _install_no_version,
 880                    venv=venv, debug=debug,
 881                ):
 882                    warn(
 883                        f"Failed to clean up package '{_install_no_version}'.",
 884                    )
 885
 886        success = run_python_package(
 887            'pip',
 888            _args + _packages,
 889            venv = venv,
 890            env = _get_pip_os_env(),
 891            debug = debug,
 892        ) == 0
 893
 894    msg = (
 895        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
 896        else "Failed to " + ('un' if _uninstall else '') + "install packages."
 897    )
 898    if not silent:
 899        print(msg)
 900    if debug:
 901        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
 902    return success
 903
 904
 905def completely_uninstall_package(
 906        install_name: str,
 907        venv: str = 'mrsm',
 908        debug: bool = False,
 909    ) -> bool:
 910    """
 911    Continue calling `pip uninstall` until a package is completely
 912    removed from a virtual environment. 
 913    This is useful for dealing with multiple installed versions of a package.
 914    """
 915    attempts = 0
 916    _install_no_version = get_install_no_version(install_name)
 917    clean_install_no_version = _install_no_version.lower().replace('-', '_')
 918    installed_versions = []
 919    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 920    if not vtp.exists():
 921        return True
 922
 923    for file_name in os.listdir(vtp):
 924        if not file_name.endswith('.dist-info'):
 925            continue
 926        clean_dist_info = file_name.replace('-', '_').lower()
 927        if not clean_dist_info.startswith(clean_install_no_version):
 928            continue
 929        installed_versions.append(file_name)
 930
 931    max_attempts = len(installed_versions) + 1
 932    while attempts < max_attempts:
 933        if not venv_contains_package(
 934            _install_to_import_name(_install_no_version),
 935            venv=venv, debug=debug,
 936        ):
 937            return True
 938        if not pip_uninstall(
 939            _install_no_version,
 940            venv=venv,
 941            silent=(not debug), debug=debug
 942        ):
 943            return False
 944        attempts += 1
 945    return False
 946
 947
 948def pip_uninstall(
 949        *args, **kw
 950    ) -> bool:
 951    """
 952    Uninstall Python packages.
 953    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
 954    """
 955    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
 956
 957
 958def run_python_package(
 959        package_name: str,
 960        args: Optional[List[str]] = None,
 961        venv: Optional[str] = 'mrsm',
 962        cwd: Optional[str] = None,
 963        foreground: bool = False,
 964        as_proc: bool = False,
 965        capture_output: bool = False,
 966        debug: bool = False,
 967        **kw: Any,
 968    ) -> Union[int, subprocess.Popen, None]:
 969    """
 970    Runs an installed python package.
 971    E.g. Translates to `/usr/bin/python -m [package]`
 972
 973    Parameters
 974    ----------
 975    package_name: str
 976        The Python module to be executed.
 977
 978    args: Optional[List[str]], default None
 979        Additional command line arguments to be appended after `-m [package]`.
 980
 981    venv: Optional[str], default 'mrsm'
 982        If specified, execute the Python interpreter from a virtual environment.
 983
 984    cwd: Optional[str], default None
 985        If specified, change directories before starting the process.
 986        Defaults to `None`.
 987
 988    as_proc: bool, default False
 989        If `True`, return a `subprocess.Popen` object.
 990
 991    capture_output: bool, default False
 992        If `as_proc` is `True`, capture stdout and stderr.
 993
 994    foreground: bool, default False
 995        If `True`, start the subprocess as a foreground process.
 996        Defaults to `False`.
 997
 998    kw: Any
 999        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1000        and by extension `subprocess.Popen()`.
1001
1002    Returns
1003    -------
1004    Either a return code integer or a `subprocess.Popen` object
1005    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1006    """
1007    import sys, platform
1008    import subprocess
1009    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1010    from meerschaum.utils.process import run_process
1011    from meerschaum.utils.warnings import warn
1012    if args is None:
1013        args = []
1014    old_cwd = os.getcwd()
1015    if cwd is not None:
1016        os.chdir(cwd)
1017    executable = venv_executable(venv=venv)
1018    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1019    import traceback
1020    if debug:
1021        print(command, file=sys.stderr)
1022    try:
1023        to_return = run_process(
1024            command,
1025            foreground = foreground,
1026            as_proc = as_proc,
1027            capture_output = capture_output,
1028            **kw
1029        )
1030    except Exception as e:
1031        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1032        warn(msg, color=False)
1033        stdout, stderr = (
1034            (None, None)
1035            if not capture_output
1036            else (subprocess.PIPE, subprocess.PIPE)
1037        )
1038        proc = subprocess.Popen(
1039            command,
1040            stdout = stdout,
1041            stderr = stderr,
1042            env = kw.get('env', os.environ),
1043        )
1044        to_return = proc if as_proc else proc.wait()
1045    except KeyboardInterrupt:
1046        to_return = 1 if not as_proc else None
1047    os.chdir(old_cwd)
1048    return to_return
1049
1050
1051def attempt_import(
1052        *names: str,
1053        lazy: bool = True,
1054        warn: bool = True,
1055        install: bool = True,
1056        venv: Optional[str] = 'mrsm',
1057        precheck: bool = True,
1058        split: bool = True,
1059        check_update: bool = False,
1060        check_pypi: bool = False,
1061        check_is_installed: bool = True,
1062        color: bool = True,
1063        debug: bool = False
1064    ) -> Union[Any, Tuple[Any]]:
1065    """
1066    Raise a warning if packages are not installed; otherwise import and return modules.
1067    If `lazy` is `True`, return lazy-imported modules.
1068    
1069    Returns tuple of modules if multiple names are provided, else returns one module.
1070    
1071    Parameters
1072    ----------
1073    names: List[str]
1074        The packages to be imported.
1075
1076    lazy: bool, default True
1077        If `True`, lazily load packages.
1078
1079    warn: bool, default True
1080        If `True`, raise a warning if a package cannot be imported.
1081
1082    install: bool, default True
1083        If `True`, attempt to install a missing package into the designated virtual environment.
1084        If `check_update` is True, install updates if available.
1085
1086    venv: Optional[str], default 'mrsm'
1087        The virtual environment in which to search for packages and to install packages into.
1088
1089    precheck: bool, default True
1090        If `True`, attempt to find module before importing (necessary for checking if modules exist
1091        and retaining lazy imports), otherwise assume lazy is `False`.
1092
1093    split: bool, default True
1094        If `True`, split packages' names on `'.'`.
1095
1096    check_update: bool, default False
1097        If `True` and `install` is `True`, install updates if the required minimum version
1098        does not match.
1099
1100    check_pypi: bool, default False
1101        If `True` and `check_update` is `True`, check PyPI when determining whether
1102        an update is required.
1103
1104    check_is_installed: bool, default True
1105        If `True`, check if the package is contained in the virtual environment.
1106
1107    Returns
1108    -------
1109    The specified modules. If they're not available and `install` is `True`, it will first
1110    download them into a virtual environment and return the modules.
1111
1112    Examples
1113    --------
1114    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1115    >>> pandas = attempt_import('pandas')
1116
1117    """
1118
1119    import importlib.util
1120
1121    ### to prevent recursion, check if parent Meerschaum package is being imported
1122    if names == ('meerschaum',):
1123        return _import_module('meerschaum')
1124
1125    if venv == 'mrsm' and _import_hook_venv is not None:
1126        if debug:
1127            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1128        venv = _import_hook_venv
1129
1130    _warnings = _import_module('meerschaum.utils.warnings')
1131    warn_function = _warnings.warn
1132
1133    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1134        with Venv(venv=venv, debug=debug):
1135            ### determine the import method (lazy vs normal)
1136            from meerschaum.utils.misc import filter_keywords
1137            import_method = (
1138                _import_module if not lazy
1139                else lazy_import
1140            )
1141            try:
1142                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1143            except Exception as e:
1144                if warn:
1145                    import traceback
1146                    traceback.print_exception(type(e), e, e.__traceback__)
1147                    warn_function(
1148                        f"Failed to import module '{_name}'.\nException:\n{e}",
1149                        ImportWarning,
1150                        stacklevel = (5 if lazy else 4),
1151                        color = False,
1152                    )
1153                mod = None
1154        return mod
1155
1156    modules = []
1157    for name in names:
1158        ### Check if package is a declared dependency.
1159        root_name = name.split('.')[0] if split else name
1160        install_name = _import_to_install_name(root_name)
1161
1162        if install_name is None:
1163            install_name = root_name
1164            if warn and root_name != 'plugins':
1165                warn_function(
1166                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1167                    ImportWarning,
1168                    stacklevel = 3,
1169                    color = False
1170                )
1171
1172        ### Determine if the package exists.
1173        if precheck is False:
1174            found_module = (
1175                do_import(
1176                    name, debug=debug, warn=False, venv=venv, color=color,
1177                    check_update=False, check_pypi=False, split=split,
1178                ) is not None
1179            )
1180        else:
1181            if check_is_installed:
1182                with _locks['_is_installed_first_check']:
1183                    if not _is_installed_first_check.get(name, False):
1184                        package_is_installed = is_installed(
1185                            name,
1186                            venv = venv,
1187                            split = split,
1188                            debug = debug,
1189                        )
1190                        _is_installed_first_check[name] = package_is_installed
1191                    else:
1192                        package_is_installed = _is_installed_first_check[name]
1193            else:
1194                package_is_installed = _is_installed_first_check.get(
1195                    name,
1196                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1197                )
1198            found_module = package_is_installed
1199
1200        if not found_module:
1201            if install:
1202                if not pip_install(
1203                    install_name,
1204                    venv = venv,
1205                    split = False,
1206                    check_update = check_update,
1207                    color = color,
1208                    debug = debug
1209                ) and warn:
1210                    warn_function(
1211                        f"Failed to install '{install_name}'.",
1212                        ImportWarning,
1213                        stacklevel = 3,
1214                        color = False,
1215                    )
1216            elif warn:
1217                ### Raise a warning if we can't find the package and install = False.
1218                warn_function(
1219                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1220                     + "some features will not work correctly."
1221                     + f"\n\nSet install=True when calling attempt_import.\n"),
1222                    ImportWarning,
1223                    stacklevel = 3,
1224                    color = False,
1225                )
1226
1227        ### Do the import. Will be lazy if lazy=True.
1228        m = do_import(
1229            name, debug=debug, warn=warn, venv=venv, color=color,
1230            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1231        )
1232        modules.append(m)
1233
1234    modules = tuple(modules)
1235    if len(modules) == 1:
1236        return modules[0]
1237    return modules
1238
1239
1240def lazy_import(
1241        name: str,
1242        local_name: str = None,
1243        **kw
1244    ) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1245    """
1246    Lazily import a package.
1247    """
1248    from meerschaum.utils.packages.lazy_loader import LazyLoader
1249    if local_name is None:
1250        local_name = name
1251    return LazyLoader(
1252        local_name,
1253        globals(),
1254        name,
1255        **kw
1256    )
1257
1258
1259def pandas_name() -> str:
1260    """
1261    Return the configured name for `pandas`.
1262    
1263    Below are the expected possible values:
1264
1265    - 'pandas'
1266    - 'modin.pandas'
1267    - 'dask.dataframe'
1268
1269    """
1270    from meerschaum.config import get_config
1271    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1272    if pandas_module_name == 'modin':
1273        pandas_module_name = 'modin.pandas'
1274    elif pandas_module_name == 'dask':
1275        pandas_module_name = 'dask.dataframe'
1276
1277    return pandas_module_name
1278
1279
1280emitted_pandas_warning: bool = False
1281def import_pandas(
1282        debug: bool = False,
1283        lazy: bool = False,
1284        **kw
1285    ) -> 'ModuleType':
1286    """
1287    Quality-of-life function to attempt to import the configured version of `pandas`.
1288    """
1289    import sys
1290    pandas_module_name = pandas_name()
1291    global emitted_pandas_warning
1292
1293    if pandas_module_name != 'pandas':
1294        with _locks['emitted_pandas_warning']:
1295            if not emitted_pandas_warning:
1296                from meerschaum.utils.warnings import warn
1297                emitted_pandas_warning = True
1298                warn(
1299                    (
1300                        "You are using an alternative Pandas implementation "
1301                        + f"'{pandas_module_name}'"
1302                        + "\n   Features may not work as expected."
1303                    ),
1304                    stack = False,
1305                )
1306
1307    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1308    pandas = attempt_import('pandas', debug=debug, lazy=False, **kw)
1309    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1310    return pd
1311
1312
1313def import_rich(
1314        lazy: bool = True,
1315        debug: bool = False,
1316        **kw : Any
1317    ) -> 'ModuleType':
1318    """
1319    Quality of life function for importing `rich`.
1320    """
1321    from meerschaum.utils.formatting import ANSI, UNICODE
1322    if not ANSI and not UNICODE:
1323        return None
1324
1325    ## need typing_extensions for `from rich import box`
1326    typing_extensions = attempt_import(
1327        'typing_extensions', lazy=False, debug=debug
1328    )
1329    pygments = attempt_import(
1330        'pygments', lazy=False,
1331    )
1332    rich = attempt_import(
1333        'rich', lazy=lazy, **kw)
1334    return rich
1335
1336
1337def _dash_less_than_2(**kw) -> bool:
1338    dash = attempt_import('dash', **kw)
1339    if dash is None:
1340        return None
1341    packaging_version = attempt_import('packaging.version', **kw)
1342    return (
1343        packaging_version.parse(dash.__version__) < 
1344        packaging_version.parse('2.0.0')
1345    )
1346
1347
1348def import_dcc(warn=False, **kw) -> 'ModuleType':
1349    """
1350    Import Dash Core Components (`dcc`).
1351    """
1352    return (
1353        attempt_import('dash_core_components', warn=warn, **kw)
1354        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1355    )
1356
1357
1358def import_html(warn=False, **kw) -> 'ModuleType':
1359    """
1360    Import Dash HTML Components (`html`).
1361    """
1362    return (
1363        attempt_import('dash_html_components', warn=warn, **kw)
1364        if _dash_less_than_2(warn=warn, **kw)
1365        else attempt_import('dash.html', warn=warn, **kw)
1366    )
1367
1368
1369def get_modules_from_package(
1370        package: 'package',
1371        names: bool = False,
1372        recursive: bool = False,
1373        lazy: bool = False,
1374        modules_venvs: bool = False,
1375        debug: bool = False
1376    ):
1377    """
1378    Find and import all modules in a package.
1379    
1380    Returns
1381    -------
1382    Either list of modules or tuple of lists.
1383    """
1384    from os.path import dirname, join, isfile, isdir, basename
1385    import glob
1386
1387    pattern = '*' if recursive else '*.py'
1388    package_path = dirname(package.__file__ or package.__path__[0])
1389    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1390    _all = [
1391        basename(f)[:-3] if isfile(f) else basename(f)
1392        for f in module_names
1393            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1394               and not f.endswith('__init__.py')
1395               and not f.endswith('__pycache__')
1396    ]
1397
1398    if debug:
1399        from meerschaum.utils.debug import dprint
1400        dprint(str(_all))
1401    modules = []
1402    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1403        ### there's probably a better way than a try: catch but it'll do for now
1404        try:
1405            ### if specified, activate the module's virtual environment before importing.
1406            ### NOTE: this only considers the filename, so two modules from different packages
1407            ### may end up sharing virtual environments.
1408            if modules_venvs:
1409                activate_venv(module_name.split('.')[-1], debug=debug)
1410            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1411            modules.append(m)
1412        except Exception as e:
1413            if debug:
1414                dprint(str(e))
1415        finally:
1416            if modules_venvs:
1417                deactivate_venv(module_name.split('.')[-1], debug=debug)
1418    if names:
1419        return _all, modules
1420
1421    return modules
1422
1423
1424def import_children(
1425        package: Optional['ModuleType'] = None,
1426        package_name: Optional[str] = None,
1427        types : Optional[List[str]] = None,
1428        lazy: bool = True,
1429        recursive: bool = False,
1430        debug: bool = False
1431    ) -> List['ModuleType']:
1432    """
1433    Import all functions in a package to its `__init__`.
1434
1435    Parameters
1436    ----------
1437    package: Optional[ModuleType], default None
1438        Package to import its functions into.
1439        If `None` (default), use parent.
1440
1441    package_name: Optional[str], default None
1442        Name of package to import its functions into
1443        If None (default), use parent.
1444
1445    types: Optional[List[str]], default None
1446        Types of members to return.
1447        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1448
1449    Returns
1450    -------
1451    A list of modules.
1452    """
1453    import sys, inspect
1454
1455    if types is None:
1456        types = ['method', 'builtin', 'function', 'class', 'module']
1457
1458    ### if package_name and package are None, use parent
1459    if package is None and package_name is None:
1460        package_name = inspect.stack()[1][0].f_globals['__name__']
1461
1462    ### populate package or package_name from other other
1463    if package is None:
1464        package = sys.modules[package_name]
1465    elif package_name is None:
1466        package_name = package.__name__
1467
1468    ### Set attributes in sys module version of package.
1469    ### Kinda like setting a dictionary
1470    ###   functions[name] = func
1471    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1472    _all, members = [], []
1473    objects = []
1474    for module in modules:
1475        _objects = []
1476        for ob in inspect.getmembers(module):
1477            for t in types:
1478                ### ob is a tuple of (name, object)
1479                if getattr(inspect, 'is' + t)(ob[1]):
1480                    _objects.append(ob)
1481
1482        if 'module' in types:
1483            _objects.append((module.__name__.split('.')[0], module))
1484        objects += _objects
1485    for ob in objects:
1486        setattr(sys.modules[package_name], ob[0], ob[1])
1487        _all.append(ob[0])
1488        members.append(ob[1])
1489
1490    if debug:
1491        from meerschaum.utils.debug import dprint
1492        dprint(str(_all))
1493    ### set __all__ for import *
1494    setattr(sys.modules[package_name], '__all__', _all)
1495    return members
1496
1497
1498_reload_module_cache = {}
1499def reload_package(
1500        package: str,
1501        skip_submodules: Optional[List[str]] = None,
1502        lazy: bool = False,
1503        debug: bool = False,
1504        **kw: Any
1505    ):
1506    """
1507    Recursively load a package's subpackages, even if they were not previously loaded.
1508    """
1509    import sys
1510    if isinstance(package, str):
1511        package_name = package
1512    else:
1513        try:
1514            package_name = package.__name__
1515        except Exception as e:
1516            package_name = str(package)
1517
1518    skip_submodules = skip_submodules or []
1519    if 'meerschaum.utils.packages' not in skip_submodules:
1520        skip_submodules.append('meerschaum.utils.packages')
1521    def safeimport():
1522        subs = [
1523            m for m in sys.modules
1524            if m.startswith(package_name + '.')
1525        ]
1526        subs_to_skip = []
1527        for skip_mod in skip_submodules:
1528            for mod in subs:
1529                if mod.startswith(skip_mod):
1530                    subs_to_skip.append(mod)
1531                    continue
1532
1533        subs = [m for m in subs if m not in subs_to_skip]
1534        for module_name in subs:
1535            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1536        if not subs_to_skip:
1537            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1538
1539        return _import_module(package_name)
1540
1541    return safeimport()
1542
1543
1544def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1545    """
1546    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1547    """
1548    reload_package(
1549        'meerschaum',
1550        skip_submodules = [
1551            'meerschaum._internal.shell',
1552            'meerschaum.utils.pool',
1553        ]
1554    )
1555
1556    from meerschaum.plugins import reload_plugins
1557    from meerschaum._internal.shell.Shell import _insert_shell_actions
1558    reload_plugins(debug=debug)
1559    _insert_shell_actions()
1560    return True, "Success"
1561
1562
1563def is_installed(
1564        import_name: str,
1565        venv: Optional[str] = 'mrsm',
1566        split: bool = True,
1567        debug: bool = False,
1568    ) -> bool:
1569    """
1570    Check whether a package is installed.
1571    """
1572    if debug:
1573        from meerschaum.utils.debug import dprint
1574    root_name = import_name.split('.')[0] if split else import_name
1575    import importlib.util
1576    with Venv(venv, debug=debug):
1577        try:
1578            spec_path = pathlib.Path(
1579                get_module_path(root_name, venv=venv, debug=debug)
1580                or
1581                importlib.util.find_spec(root_name).origin
1582            )
1583        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1584            spec_path = None
1585
1586        found = (
1587            not need_update(
1588                None, import_name = root_name,
1589                _run_determine_version = False,
1590                check_pypi = False,
1591                version = determine_version(
1592                    spec_path, venv=venv, debug=debug, import_name=root_name
1593                ),
1594                debug = debug,
1595            )
1596        ) if spec_path is not None else False
1597
1598    return found
1599
1600
1601def venv_contains_package(
1602        import_name: str,
1603        venv: Optional[str] = 'mrsm',
1604        split: bool = True,
1605        debug: bool = False,
1606    ) -> bool:
1607    """
1608    Search the contents of a virtual environment for a package.
1609    """
1610    root_name = import_name.split('.')[0] if split else import_name
1611    return get_module_path(root_name, venv=venv, debug=debug) is not None
1612
1613
1614def package_venv(package: 'ModuleType') -> Union[str, None]:
1615    """
1616    Inspect a package and return the virtual environment in which it presides.
1617    """
1618    import os
1619    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1620    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1621        return None
1622    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
1623
1624
1625def ensure_readline() -> 'ModuleType':
1626    """Make sure that the `readline` package is able to be imported."""
1627    import sys
1628    try:
1629        import readline
1630    except ImportError:
1631        readline = None
1632
1633    if readline is None:
1634        import platform
1635        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1636        try:
1637            rl = attempt_import(
1638                rl_name,
1639                lazy = False,
1640                install = True,
1641                venv = None,
1642                warn = False,
1643            )
1644        except (ImportError, ModuleNotFoundError):
1645            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1646                print(f"Unable to import {rl_name}!", file=sys.stderr)
1647                sys.exit(1)
1648
1649    sys.modules['readline'] = readline
1650    return readline
1651
1652_pkg_resources_get_distribution = None
1653_custom_distributions = {}
1654def _monkey_patch_get_distribution(_dist: str, _version: str) -> None:
1655    """
1656    Monkey patch `pkg_resources.get_distribution` to allow for importing `flask_compress`.
1657    """
1658    import pkg_resources
1659    from collections import namedtuple
1660    global _pkg_resources_get_distribution
1661    with _locks['_pkg_resources_get_distribution']:
1662        _pkg_resources_get_distribution = pkg_resources.get_distribution
1663    _custom_distributions[_dist] = _version
1664    _Dist = namedtuple('_Dist', ['version'])
1665    def _get_distribution(dist):
1666        """Hack for flask-compress."""
1667        if dist in _custom_distributions:
1668            return _Dist(_custom_distributions[dist])
1669        return _pkg_resources_get_distribution(dist)
1670    pkg_resources.get_distribution = _get_distribution
1671
1672
1673def _get_pip_os_env():
1674    """
1675    Return the environment variables context in which `pip` should be run.
1676    See PEP 668 for why we are overriding the environment.
1677    """
1678    import os
1679    pip_os_env = os.environ.copy()
1680    pip_os_env.update({
1681        'PIP_BREAK_SYSTEM_PACKAGES': 'true',
1682    })
1683    return pip_os_env
def get_module_path( import_name: str, venv: Optional[str] = 'mrsm', debug: bool = False, _try_install_name_on_fail: bool = True) -> Optional[pathlib.Path]:
 40def get_module_path(
 41        import_name: str,
 42        venv: Optional[str] = 'mrsm',
 43        debug: bool = False,
 44        _try_install_name_on_fail: bool = True,
 45    ) -> Union[pathlib.Path, None]:
 46    """
 47    Get a module's path without importing.
 48    """
 49    if debug:
 50        from meerschaum.utils.debug import dprint
 51    if not _try_install_name_on_fail:
 52        install_name = _import_to_install_name(import_name, with_version=False)
 53        install_name_lower = install_name.lower().replace('-', '_')
 54        import_name_lower = install_name_lower
 55    else:
 56        import_name_lower = import_name.lower().replace('-', '_')
 57    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
 58    if not vtp.exists():
 59        if debug:
 60            dprint(f"Venv '{venv}' does not exist, cannot import '{import_name}'.", color=False)
 61        return None
 62    candidates = []
 63    for file_name in os.listdir(vtp):
 64        file_name_lower = file_name.lower().replace('-', '_')
 65        if not file_name_lower.startswith(import_name_lower):
 66            continue
 67        if file_name.endswith('dist_info'):
 68            continue
 69        file_path = vtp / file_name
 70
 71        ### Most likely: Is a directory with __init__.py
 72        if file_name_lower == import_name_lower and file_path.is_dir():
 73            init_path = file_path / '__init__.py'
 74            if init_path.exists():
 75                candidates.append(init_path)
 76
 77        ### May be a standalone .py file.
 78        elif file_name_lower == import_name_lower + '.py':
 79            candidates.append(file_path)
 80
 81        ### Compiled wheels (e.g. pyodbc)
 82        elif file_name_lower.startswith(import_name_lower + '.'):
 83            candidates.append(file_path)
 84
 85    if len(candidates) == 1:
 86        return candidates[0]
 87
 88    if not candidates:
 89        if _try_install_name_on_fail:
 90            return get_module_path(
 91                import_name, venv=venv, debug=debug,
 92                _try_install_name_on_fail=False
 93            )
 94        return None
 95
 96    specs_paths = []
 97    for candidate_path in candidates:
 98        spec = importlib.util.spec_from_file_location(import_name, str(candidate_path))
 99        if spec is not None:
100            return candidate_path
101    
102    return None

Get a module's path without importing.

def manually_import_module( import_name: str, venv: Optional[str] = 'mrsm', check_update: bool = True, check_pypi: bool = False, install: bool = True, split: bool = True, warn: bool = True, color: bool = True, debug: bool = False, use_sys_modules: bool = True) -> "Union['ModuleType', None]":
105def manually_import_module(
106        import_name: str,
107        venv: Optional[str] = 'mrsm',
108        check_update: bool = True,
109        check_pypi: bool = False,
110        install: bool = True,
111        split: bool = True,
112        warn: bool = True,
113        color: bool = True,
114        debug: bool = False,
115        use_sys_modules: bool = True,
116    ) -> Union['ModuleType', None]:
117    """
118    Manually import a module from a virtual environment (or the base environment).
119
120    Parameters
121    ----------
122    import_name: str
123        The name of the module.
124        
125    venv: Optional[str], default 'mrsm'
126        The virtual environment to read from.
127
128    check_update: bool, default True
129        If `True`, examine whether the available version of the package meets the required version.
130
131    check_pypi: bool, default False
132        If `True`, check PyPI for updates before importing.
133
134    install: bool, default True
135        If `True`, install the package if it's not installed or needs an update.
136
137    split: bool, default True
138        If `True`, split `import_name` on periods to get the package name.
139
140    warn: bool, default True
141        If `True`, raise a warning if the package cannot be imported.
142
143    color: bool, default True
144        If `True`, use color output for debug and warning text.
145
146    debug: bool, default False
147        Verbosity toggle.
148
149    use_sys_modules: bool, default True
150        If `True`, return the module in `sys.modules` if it exists.
151        Otherwise continue with manually importing.
152
153    Returns
154    -------
155    The specified module or `None` if it can't be imported.
156
157    """
158    import sys
159    _previously_imported = import_name in sys.modules
160    if _previously_imported and use_sys_modules:
161        return sys.modules[import_name]
162    if debug:
163        from meerschaum.utils.debug import dprint
164    from meerschaum.utils.warnings import warn as warn_function
165    import warnings
166    root_name = import_name.split('.')[0] if split else import_name
167    install_name = _import_to_install_name(root_name)
168
169    root_path = get_module_path(root_name, venv=venv)
170    if root_path is None:
171        return None
172    mod_path = root_path
173    if mod_path.is_dir():
174        for _dir in import_name.split('.')[:-1]:
175            mod_path = mod_path / _dir
176            possible_end_module_filename = import_name.split('.')[-1] + '.py'
177            try:
178                mod_path = (
179                    (mod_path / possible_end_module_filename)
180                    if possible_end_module_filename in os.listdir(mod_path)
181                    else (
182                        mod_path / import_name.split('.')[-1] / '__init__.py'
183                    )
184                )
185            except Exception as e:
186                mod_path = None
187
188    spec = (
189        importlib.util.find_spec(import_name) if mod_path is None or not mod_path.exists()
190        else importlib.util.spec_from_file_location(import_name, str(mod_path))
191    )
192    root_spec = (
193        importlib.util.find_spec(root_name) if not root_path.exists()
194        else importlib.util.spec_from_file_location(root_name, str(root_path))
195    )
196
197    ### Check for updates before importing.
198    _version = (
199        determine_version(
200            pathlib.Path(root_spec.origin),
201            import_name=root_name, venv=venv, debug=debug
202        ) if root_spec is not None and root_spec.origin is not None else None
203    )
204
205    if _version is not None:
206        if check_update:
207            if need_update(
208                None,
209                import_name = root_name,
210                version = _version,
211                check_pypi = check_pypi,
212                debug = debug,
213            ):
214                if install:
215                    if not pip_install(
216                        root_name,
217                        venv = venv,
218                        split = False,
219                        check_update = check_update,
220                        color = color,
221                        debug = debug
222                    ) and warn:
223                        warn_function(
224                            f"There's an update available for '{install_name}', "
225                            + "but it failed to install. "
226                            + "Try installig via Meerschaum with "
227                            + "`install packages '{install_name}'`.",
228                            ImportWarning,
229                            stacklevel = 3,
230                            color = False,
231                        )
232                elif warn:
233                    warn_function(
234                        f"There's an update available for '{root_name}'.",
235                        stack = False,
236                        color = False,
237                    )
238                spec = (
239                    importlib.util.find_spec(import_name)
240                    if mod_path is None or not mod_path.exists()
241                    else importlib.util.spec_from_file_location(import_name, str(mod_path))
242                )
243
244
245    if spec is None:
246        try:
247            mod = _import_module(import_name)
248        except Exception as e:
249            mod = None
250        return mod
251
252    with Venv(venv, debug=debug):
253        mod = importlib.util.module_from_spec(spec)
254        old_sys_mod = sys.modules.get(import_name, None)
255        sys.modules[import_name] = mod
256
257        try:
258            with warnings.catch_warnings():
259                warnings.filterwarnings('ignore', 'The NumPy')
260                spec.loader.exec_module(mod)
261        except Exception as e:
262            pass
263        mod = _import_module(import_name)
264        if old_sys_mod is not None:
265            sys.modules[import_name] = old_sys_mod
266        else:
267            del sys.modules[import_name]
268    
269    return mod

Manually import a module from a virtual environment (or the base environment).

Parameters
  • import_name (str): The name of the module.
  • venv (Optional[str], default 'mrsm'): The virtual environment to read from.
  • check_update (bool, default True): If True, examine whether the available version of the package meets the required version.
  • check_pypi (bool, default False): If True, check PyPI for updates before importing.
  • install (bool, default True): If True, install the package if it's not installed or needs an update.
  • split (bool, default True): If True, split import_name on periods to get the package name.
  • warn (bool, default True): If True, raise a warning if the package cannot be imported.
  • color (bool, default True): If True, use color output for debug and warning text.
  • debug (bool, default False): Verbosity toggle.
  • use_sys_modules (bool, default True): If True, return the module in sys.modules if it exists. Otherwise continue with manually importing.
Returns
  • The specified module or None if it can't be imported.
def get_install_no_version(install_name: str) -> str:
300def get_install_no_version(install_name: str) -> str:
301    """
302    Strip the version information from the install name.
303    """
304    import re
305    return re.split(r'[\[=<>,! \]]', install_name)[0]

Strip the version information from the install name.

import_versions = {'mrsm': {'daemon': '3.0.1', 'packaging': '23.2', 'semver': '3.0.2', 'prompt_toolkit': '3.0.43'}}
def determine_version( path: pathlib.Path, import_name: Optional[str] = None, venv: Optional[str] = 'mrsm', search_for_metadata: bool = True, split: bool = True, warn: bool = False, debug: bool = False) -> Optional[str]:
309def determine_version(
310        path: pathlib.Path,
311        import_name: Optional[str] = None,
312        venv: Optional[str] = 'mrsm',
313        search_for_metadata: bool = True,
314        split: bool = True,
315        warn: bool = False,
316        debug: bool = False,
317    ) -> Union[str, None]:
318    """
319    Determine a module's `__version__` string from its filepath.
320    
321    First it searches for pip metadata, then it attempts to import the module in a subprocess.
322
323    Parameters
324    ----------
325    path: pathlib.Path
326        The file path of the module.
327
328    import_name: Optional[str], default None
329        The name of the module. If omitted, it will be determined from the file path.
330        Defaults to `None`.
331
332    venv: Optional[str], default 'mrsm'
333        The virtual environment of the Python interpreter to use if importing is necessary.
334
335    search_for_metadata: bool, default True
336        If `True`, search the pip site_packages directory (assumed to be the parent)
337        for the corresponding dist-info directory.
338
339    warn: bool, default True
340        If `True`, raise a warning if the module fails to import in the subprocess.
341
342    split: bool, default True
343        If `True`, split the determined import name by periods to get the room name.
344
345    Returns
346    -------
347    The package's version string if available or `None`.
348    If multiple versions are found, it will trigger an import in a subprocess.
349
350    """
351    with _locks['import_versions']:
352        if venv not in import_versions:
353            import_versions[venv] = {}
354    import importlib.metadata
355    import re, os
356    old_cwd = os.getcwd()
357    if debug:
358        from meerschaum.utils.debug import dprint
359    from meerschaum.utils.warnings import warn as warn_function
360    if import_name is None:
361        import_name = path.parent.stem if path.stem == '__init__' else path.stem
362        import_name = import_name.split('.')[0] if split else import_name
363    if import_name in import_versions[venv]:
364        return import_versions[venv][import_name]
365    _version = None
366    module_parent_dir = (
367        path.parent.parent if path.stem == '__init__' else path.parent
368    ) if path is not None else venv_target_path(venv, debug=debug)
369
370    installed_dir_name = _import_to_dir_name(import_name)
371    clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
372
373    ### First, check if a dist-info directory exists.
374    _found_versions = []
375    if search_for_metadata:
376        for filename in os.listdir(module_parent_dir):
377            if not filename.endswith('.dist-info'):
378                continue
379            filename_lower = filename.lower()
380            if not filename_lower.startswith(clean_installed_dir_name + '-'):
381                continue
382            _v = filename.replace('.dist-info', '').split("-")[-1]
383            _found_versions.append(_v)
384
385    if len(_found_versions) == 1:
386        _version = _found_versions[0]
387        with _locks['import_versions']:
388            import_versions[venv][import_name] = _version
389        return _found_versions[0]
390
391    if not _found_versions:
392        try:
393            import importlib.metadata as importlib_metadata
394        except ImportError:
395            importlib_metadata = attempt_import(
396                'importlib_metadata',
397                debug=debug, check_update=False, precheck=False,
398                color=False, check_is_installed=False, lazy=False,
399            )
400        try:
401            os.chdir(module_parent_dir)
402            _version = importlib_metadata.metadata(import_name)['Version']
403        except Exception as e:
404            _version = None
405        finally:
406            os.chdir(old_cwd)
407
408        if _version is not None:
409            with _locks['import_versions']:
410                import_versions[venv][import_name] = _version
411            return _version
412
413    if debug:
414        print(f'Found multiple versions for {import_name}: {_found_versions}')
415
416    module_parent_dir_str = module_parent_dir.as_posix()
417
418    ### Not a pip package, so let's try importing the module directly (in a subprocess).
419    _no_version_str = 'no-version'
420    code = (
421        f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n"
422        + f"module = importlib.import_module('{import_name}');\n"
423        + "try:\n"
424        + "  print(module.__version__ , end='')\n"
425        + "except:\n"
426        + f"  print('{_no_version_str}', end='')"
427    )
428    exit_code, stdout_bytes, stderr_bytes = venv_exec(
429        code, venv=venv, with_extras=True, debug=debug
430    )
431    stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8')
432    _version = stdout.split('\n')[-1] if exit_code == 0 else None
433    _version = _version if _version != _no_version_str else None
434
435    if _version is None:
436        _version = _get_package_metadata(import_name, venv).get('version', None)
437    if _version is None and warn:
438        warn_function(
439            f"Failed to determine a version for '{import_name}':\n{stderr}",
440            stack = False
441        )
442
443    ### If `__version__` doesn't exist, return `None`.
444    import_versions[venv][import_name] = _version
445    return _version

Determine a module's __version__ string from its filepath.

First it searches for pip metadata, then it attempts to import the module in a subprocess.

Parameters
  • path (pathlib.Path): The file path of the module.
  • import_name (Optional[str], default None): The name of the module. If omitted, it will be determined from the file path. Defaults to None.
  • venv (Optional[str], default 'mrsm'): The virtual environment of the Python interpreter to use if importing is necessary.
  • search_for_metadata (bool, default True): If True, search the pip site_packages directory (assumed to be the parent) for the corresponding dist-info directory.
  • warn (bool, default True): If True, raise a warning if the module fails to import in the subprocess.
  • split (bool, default True): If True, split the determined import name by periods to get the room name.
Returns
  • The package's version string if available or None.
  • If multiple versions are found, it will trigger an import in a subprocess.
def need_update( package: "Optional['ModuleType']" = None, install_name: Optional[str] = None, import_name: Optional[str] = None, version: Optional[str] = None, check_pypi: bool = False, split: bool = True, color: bool = True, debug: bool = False, _run_determine_version: bool = True) -> bool:
490def need_update(
491        package: Optional['ModuleType'] = None,
492        install_name: Optional[str] = None,
493        import_name: Optional[str] = None,
494        version: Optional[str] = None,
495        check_pypi: bool = False,
496        split: bool = True,
497        color: bool = True,
498        debug: bool = False,
499        _run_determine_version: bool = True,
500    ) -> bool:
501    """
502    Check if a Meerschaum dependency needs an update.
503    Returns a bool for whether or not a package needs to be updated.
504
505    Parameters
506    ----------
507    package: 'ModuleType'
508        The module of the package to be updated.
509
510    install_name: Optional[str], default None
511        If provided, use this string to determine the required version.
512        Otherwise use the install name defined in `meerschaum.utils.packages._packages`.
513
514    import_name:
515        If provided, override the package's `__name__` string.
516
517    version: Optional[str], default None
518        If specified, override the package's `__version__` string.
519
520    check_pypi: bool, default False
521        If `True`, check pypi.org for updates.
522        Defaults to `False`.
523
524    split: bool, default True
525        If `True`, split the module's name on periods to detrive the root name.
526        Defaults to `True`.
527
528    color: bool, default True
529        If `True`, format debug output.
530        Defaults to `True`.
531
532    debug: bool, default True
533        Verbosity toggle.
534
535    Returns
536    -------
537    A bool indicating whether the package requires an update.
538
539    """
540    if debug:
541        from meerschaum.utils.debug import dprint
542    from meerschaum.utils.warnings import warn as warn_function
543    import re
544    root_name = (
545        package.__name__.split('.')[0] if split else package.__name__
546    ) if import_name is None else (
547        import_name.split('.')[0] if split else import_name
548    )
549    install_name = install_name or _import_to_install_name(root_name)
550    with _locks['_checked_for_updates']:
551        if install_name in _checked_for_updates:
552            return False
553        _checked_for_updates.add(install_name)
554
555    _install_no_version = get_install_no_version(install_name)
556    required_version = install_name.replace(_install_no_version, '')
557    if ']' in required_version:
558        required_version = required_version.split(']')[1]
559
560    ### No minimum version was specified, and we're not going to check PyPI.
561    if not required_version and not check_pypi:
562        return False
563
564    ### NOTE: Sometimes (rarely), we depend on a development build of a package.
565    if '.dev' in required_version:
566        required_version = required_version.split('.dev')[0]
567    if version and '.dev' in version:
568        version = version.split('.dev')[0]
569
570    try:
571        if not version:
572            if not _run_determine_version:
573                version = determine_version(
574                    pathlib.Path(package.__file__),
575                    import_name=root_name, warn=False, debug=debug
576                )
577        if version is None:
578            return False
579    except Exception as e:
580        if debug:
581            dprint(str(e), color=color)
582            dprint("No version could be determined from the installed package.", color=color)
583        return False
584    split_version = version.split('.')
585    last_part = split_version[-1]
586    if len(split_version) == 2:
587        version = '.'.join(split_version) + '.0'
588    elif 'dev' in last_part or 'rc' in last_part:
589        tag = 'dev' if 'dev' in last_part else 'rc'
590        last_sep = '-'
591        if not last_part.startswith(tag):
592            last_part = f'-{tag}'.join(last_part.split(tag))
593            last_sep = '.'
594        version = '.'.join(split_version[:-1]) + last_sep + last_part
595    elif len(split_version) > 3:
596        version = '.'.join(split_version[:3])
597
598    packaging_version = attempt_import(
599        'packaging.version', check_update=False, lazy=False, debug=debug,
600    )
601
602    ### Get semver if necessary
603    if required_version:
604        semver_path = get_module_path('semver', debug=debug)
605        if semver_path is None:
606            pip_install(_import_to_install_name('semver'), debug=debug)
607        semver = attempt_import('semver', check_update=False, lazy=False, debug=debug)
608    if check_pypi:
609        ### Check PyPI for updates
610        update_checker = attempt_import(
611            'update_checker', lazy=False, check_update=False, debug=debug
612        )
613        checker = update_checker.UpdateChecker()
614        result = checker.check(_install_no_version, version)
615    else:
616        ### Skip PyPI and assume we can't be sure.
617        result = None
618
619    ### Compare PyPI's version with our own.
620    if result is not None:
621        ### We have a result from PyPI and a stated required version.
622        if required_version:
623            try:
624                return semver.Version.parse(result.available_version).match(required_version)
625            except AttributeError as e:
626                pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
627                semver = manually_import_module('semver', venv='mrsm')
628                return semver.Version.parse(version).match(required_version)
629            except Exception as e:
630                if debug:
631                    dprint(f"Failed to match versions with exception:\n{e}", color=color)
632                return False
633
634        ### If `check_pypi` and we don't have a required version, check if PyPI's version
635        ### is newer than the installed version.
636        else:
637            return (
638                packaging_version.parse(result.available_version) > 
639                packaging_version.parse(version)
640            )
641
642    ### We might be depending on a prerelease.
643    ### Sanity check that the required version is not greater than the installed version. 
644    try:
645        return (
646            (not semver.Version.parse(version).match(required_version))
647            if required_version else False
648        )
649    except AttributeError as e:
650        pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
651        semver = manually_import_module('semver', venv='mrsm', debug=debug)
652        return (
653            (not semver.Version.parse(version).match(required_version))
654            if required_version else False
655        )
656    except Exception as e:
657        print(f"Unable to parse version ({version}) for package '{import_name}'.")
658        print(e)
659        if debug:
660            dprint(e)
661        return False
662    try:
663        return (
664            packaging_version.parse(version) > 
665            packaging_version.parse(required_version)
666        )
667    except Exception as e:
668        if debug:
669            dprint(e)
670        return False
671    return False

Check if a Meerschaum dependency needs an update. Returns a bool for whether or not a package needs to be updated.

Parameters
  • package ('ModuleType'): The module of the package to be updated.
  • install_name (Optional[str], default None): If provided, use this string to determine the required version. Otherwise use the install name defined in meerschaum.utils.packages._packages.
  • import_name:: If provided, override the package's __name__ string.
  • version (Optional[str], default None): If specified, override the package's __version__ string.
  • check_pypi (bool, default False): If True, check pypi.org for updates. Defaults to False.
  • split (bool, default True): If True, split the module's name on periods to detrive the root name. Defaults to True.
  • color (bool, default True): If True, format debug output. Defaults to True.
  • debug (bool, default True): Verbosity toggle.
Returns
  • A bool indicating whether the package requires an update.
def get_pip(venv: Optional[str] = 'mrsm', debug: bool = False) -> bool:
674def get_pip(venv: Optional[str] = 'mrsm', debug: bool=False) -> bool:
675    """
676    Download and run the get-pip.py script.
677
678    Parameters
679    ----------
680    debug: bool, default False
681        Verbosity toggle.
682
683    Returns
684    -------
685    A bool indicating success.
686
687    """
688    import sys, subprocess
689    from meerschaum.utils.misc import wget
690    from meerschaum.config._paths import CACHE_RESOURCES_PATH
691    from meerschaum.config.static import STATIC_CONFIG
692    url = STATIC_CONFIG['system']['urls']['get-pip.py']
693    dest = CACHE_RESOURCES_PATH / 'get-pip.py'
694    try:
695        wget(url, dest, color=False, debug=debug)
696    except Exception as e:
697        print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
698        sys.exit(1)
699    if venv is not None:
700        init_venv(venv=venv, debug=debug)
701    cmd_list = [venv_executable(venv=venv), dest.as_posix()] 
702    return subprocess.call(cmd_list, env=_get_pip_os_env()) == 0

Download and run the get-pip.py script.

Parameters
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating success.
def pip_install( *install_names: str, args: Optional[List[str]] = None, requirements_file_path: Union[pathlib.Path, str, NoneType] = None, venv: Optional[str] = 'mrsm', split: bool = False, check_update: bool = True, check_pypi: bool = True, check_wheel: bool = True, _uninstall: bool = False, color: bool = True, silent: bool = False, debug: bool = False) -> bool:
705def pip_install(
706        *install_names: str,
707        args: Optional[List[str]] = None,
708        requirements_file_path: Union[pathlib.Path, str, None] = None,
709        venv: Optional[str] = 'mrsm',
710        split: bool = False,
711        check_update: bool = True,
712        check_pypi: bool = True,
713        check_wheel: bool = True,
714        _uninstall: bool = False,
715        color: bool = True,
716        silent: bool = False,
717        debug: bool = False,
718    ) -> bool:
719    """
720    Install packages from PyPI with `pip`.
721
722    Parameters
723    ----------
724    *install_names: str
725        The installation names of packages to be installed.
726        This includes version restrictions.
727        Use `_import_to_install_name()` to get the predefined `install_name` for a package
728        from its import name.
729        
730    args: Optional[List[str]], default None
731        A list of command line arguments to pass to `pip`.
732        If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
733
734    requirements_file_path: Optional[pathlib.Path, str], default None
735        If provided, append `['-r', '/path/to/requirements.txt']` to `args`.
736
737    venv: str, default 'mrsm'
738        The virtual environment to install into.
739
740    split: bool, default False
741        If `True`, split on periods and only install the root package name.
742
743    check_update: bool, default True
744        If `True`, check if the package requires an update.
745
746    check_pypi: bool, default True
747        If `True` and `check_update` is `True`, check PyPI for the latest version.
748
749    check_wheel: bool, default True
750        If `True`, check if `wheel` is available.
751
752    _uninstall: bool, default False
753        If `True`, uninstall packages instead.
754
755    color: bool, default True
756        If `True`, include color in debug text.
757
758    silent: bool, default False
759        If `True`, skip printing messages.
760
761    debug: bool, default False
762        Verbosity toggle.
763
764    Returns
765    -------
766    A bool indicating success.
767
768    """
769    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
770    from meerschaum.utils.warnings import warn
771    if args is None:
772        args = ['--upgrade'] if not _uninstall else []
773    if color:
774        ANSI, UNICODE = True, True
775    else:
776        ANSI, UNICODE = False, False
777    if check_wheel:
778        have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
779
780    _args = list(args)
781    have_pip = venv_contains_package('pip', venv=venv, debug=debug)
782    import sys
783    if not have_pip:
784        if not get_pip(venv=venv, debug=debug):
785            import sys
786            minor = sys.version_info.minor
787            print(
788                "\nFailed to import `pip` and `ensurepip`.\n"
789                + "If you are running Ubuntu/Debian, "
790                + "you might need to install `python3.{minor}-distutils`:\n\n"
791                + f"    sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n"
792                + "Please install pip and restart Meerschaum.\n\n"
793                + "You can find instructions on installing `pip` here:\n"
794                + "https://pip.pypa.io/en/stable/installing/"
795            )
796            sys.exit(1)
797    
798    with Venv(venv, debug=debug):
799        if venv is not None:
800            if '--ignore-installed' not in args and '-I' not in _args and not _uninstall:
801                _args += ['--ignore-installed']
802            if '--cache-dir' not in args and not _uninstall:
803                cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
804                _args += ['--cache-dir', str(cache_dir_path)]
805
806        if 'pip' not in ' '.join(_args):
807            if check_update and not _uninstall:
808                pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False)
809                if need_update(pip, check_pypi=check_pypi, debug=debug):
810                    _args.append(all_packages['pip'])
811        
812        _args = (['install'] if not _uninstall else ['uninstall']) + _args
813
814        if check_wheel and not _uninstall:
815            if not have_wheel:
816                if not pip_install(
817                    'setuptools', 'wheel',
818                    venv = venv,
819                    check_update = False, check_pypi = False,
820                    check_wheel = False, debug = debug,
821                ):
822                    warn(
823                        f"Failed to install `setuptools` and `wheel` for virtual environment '{venv}'.",
824                        color=False,
825                    )
826
827        if requirements_file_path is not None:
828            _args.append('-r')
829            _args.append(str(pathlib.Path(requirements_file_path).resolve()))
830
831        if not ANSI and '--no-color' not in _args:
832            _args.append('--no-color')
833
834        if '--no-input' not in _args:
835            _args.append('--no-input')
836
837        if _uninstall and '-y' not in _args:
838            _args.append('-y')
839
840        if '--no-warn-conflicts' not in _args and not _uninstall:
841            _args.append('--no-warn-conflicts')
842
843        if '--disable-pip-version-check' not in _args:
844            _args.append('--disable-pip-version-check')
845
846        if '--target' not in _args and '-t' not in _args and not _uninstall:
847            if venv is not None:
848                _args += ['--target', venv_target_path(venv, debug=debug)]
849        elif (
850            '--target' not in _args
851                and '-t' not in _args
852                and not inside_venv()
853                and not _uninstall
854        ):
855            _args += ['--user']
856
857        if debug:
858            if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args:
859                pass
860        else:
861            if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args:
862                pass
863
864        _packages = [
865            (install_name if not _uninstall else get_install_no_version(install_name))
866            for install_name in install_names
867        ]
868        msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
869        for p in _packages:
870            msg += f'\n  - {p}'
871        if not silent:
872            print(msg)
873
874        if not _uninstall:
875            for install_name in _packages:
876                _install_no_version = get_install_no_version(install_name)
877                if _install_no_version in ('pip', 'wheel'):
878                    continue
879                if not completely_uninstall_package(
880                    _install_no_version,
881                    venv=venv, debug=debug,
882                ):
883                    warn(
884                        f"Failed to clean up package '{_install_no_version}'.",
885                    )
886
887        success = run_python_package(
888            'pip',
889            _args + _packages,
890            venv = venv,
891            env = _get_pip_os_env(),
892            debug = debug,
893        ) == 0
894
895    msg = (
896        "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
897        else "Failed to " + ('un' if _uninstall else '') + "install packages."
898    )
899    if not silent:
900        print(msg)
901    if debug:
902        print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
903    return success

Install packages from PyPI with pip.

Parameters
  • *install_names (str): The installation names of packages to be installed. This includes version restrictions. Use _import_to_install_name() to get the predefined install_name for a package from its import name.
  • args (Optional[List[str]], default None): A list of command line arguments to pass to pip. If not provided, default to ['--upgrade'] if _uninstall is False, else [].
  • requirements_file_path (Optional[pathlib.Path, str], default None): If provided, append ['-r', '/path/to/requirements.txt'] to args.
  • venv (str, default 'mrsm'): The virtual environment to install into.
  • split (bool, default False): If True, split on periods and only install the root package name.
  • check_update (bool, default True): If True, check if the package requires an update.
  • check_pypi (bool, default True): If True and check_update is True, check PyPI for the latest version.
  • check_wheel (bool, default True): If True, check if wheel is available.
  • _uninstall (bool, default False): If True, uninstall packages instead.
  • color (bool, default True): If True, include color in debug text.
  • silent (bool, default False): If True, skip printing messages.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating success.
def completely_uninstall_package(install_name: str, venv: str = 'mrsm', debug: bool = False) -> bool:
906def completely_uninstall_package(
907        install_name: str,
908        venv: str = 'mrsm',
909        debug: bool = False,
910    ) -> bool:
911    """
912    Continue calling `pip uninstall` until a package is completely
913    removed from a virtual environment. 
914    This is useful for dealing with multiple installed versions of a package.
915    """
916    attempts = 0
917    _install_no_version = get_install_no_version(install_name)
918    clean_install_no_version = _install_no_version.lower().replace('-', '_')
919    installed_versions = []
920    vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
921    if not vtp.exists():
922        return True
923
924    for file_name in os.listdir(vtp):
925        if not file_name.endswith('.dist-info'):
926            continue
927        clean_dist_info = file_name.replace('-', '_').lower()
928        if not clean_dist_info.startswith(clean_install_no_version):
929            continue
930        installed_versions.append(file_name)
931
932    max_attempts = len(installed_versions) + 1
933    while attempts < max_attempts:
934        if not venv_contains_package(
935            _install_to_import_name(_install_no_version),
936            venv=venv, debug=debug,
937        ):
938            return True
939        if not pip_uninstall(
940            _install_no_version,
941            venv=venv,
942            silent=(not debug), debug=debug
943        ):
944            return False
945        attempts += 1
946    return False

Continue calling pip uninstall until a package is completely removed from a virtual environment. This is useful for dealing with multiple installed versions of a package.

def pip_uninstall(*args, **kw) -> bool:
949def pip_uninstall(
950        *args, **kw
951    ) -> bool:
952    """
953    Uninstall Python packages.
954    This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`.
955    """
956    return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})

Uninstall Python packages. This function is a wrapper around pip_install() but with _uninstall enforced as True.

def run_python_package( package_name: str, args: Optional[List[str]] = None, venv: Optional[str] = 'mrsm', cwd: Optional[str] = None, foreground: bool = False, as_proc: bool = False, capture_output: bool = False, debug: bool = False, **kw: Any) -> Union[int, subprocess.Popen, NoneType]:
 959def run_python_package(
 960        package_name: str,
 961        args: Optional[List[str]] = None,
 962        venv: Optional[str] = 'mrsm',
 963        cwd: Optional[str] = None,
 964        foreground: bool = False,
 965        as_proc: bool = False,
 966        capture_output: bool = False,
 967        debug: bool = False,
 968        **kw: Any,
 969    ) -> Union[int, subprocess.Popen, None]:
 970    """
 971    Runs an installed python package.
 972    E.g. Translates to `/usr/bin/python -m [package]`
 973
 974    Parameters
 975    ----------
 976    package_name: str
 977        The Python module to be executed.
 978
 979    args: Optional[List[str]], default None
 980        Additional command line arguments to be appended after `-m [package]`.
 981
 982    venv: Optional[str], default 'mrsm'
 983        If specified, execute the Python interpreter from a virtual environment.
 984
 985    cwd: Optional[str], default None
 986        If specified, change directories before starting the process.
 987        Defaults to `None`.
 988
 989    as_proc: bool, default False
 990        If `True`, return a `subprocess.Popen` object.
 991
 992    capture_output: bool, default False
 993        If `as_proc` is `True`, capture stdout and stderr.
 994
 995    foreground: bool, default False
 996        If `True`, start the subprocess as a foreground process.
 997        Defaults to `False`.
 998
 999    kw: Any
1000        Additional keyword arguments to pass to `meerschaum.utils.process.run_process()`
1001        and by extension `subprocess.Popen()`.
1002
1003    Returns
1004    -------
1005    Either a return code integer or a `subprocess.Popen` object
1006    (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
1007    """
1008    import sys, platform
1009    import subprocess
1010    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1011    from meerschaum.utils.process import run_process
1012    from meerschaum.utils.warnings import warn
1013    if args is None:
1014        args = []
1015    old_cwd = os.getcwd()
1016    if cwd is not None:
1017        os.chdir(cwd)
1018    executable = venv_executable(venv=venv)
1019    command = [executable, '-m', str(package_name)] + [str(a) for a in args]
1020    import traceback
1021    if debug:
1022        print(command, file=sys.stderr)
1023    try:
1024        to_return = run_process(
1025            command,
1026            foreground = foreground,
1027            as_proc = as_proc,
1028            capture_output = capture_output,
1029            **kw
1030        )
1031    except Exception as e:
1032        msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
1033        warn(msg, color=False)
1034        stdout, stderr = (
1035            (None, None)
1036            if not capture_output
1037            else (subprocess.PIPE, subprocess.PIPE)
1038        )
1039        proc = subprocess.Popen(
1040            command,
1041            stdout = stdout,
1042            stderr = stderr,
1043            env = kw.get('env', os.environ),
1044        )
1045        to_return = proc if as_proc else proc.wait()
1046    except KeyboardInterrupt:
1047        to_return = 1 if not as_proc else None
1048    os.chdir(old_cwd)
1049    return to_return

Runs an installed python package. E.g. Translates to /usr/bin/python -m [package]

Parameters
  • package_name (str): The Python module to be executed.
  • args (Optional[List[str]], default None): Additional command line arguments to be appended after -m [package].
  • venv (Optional[str], default 'mrsm'): If specified, execute the Python interpreter from a virtual environment.
  • cwd (Optional[str], default None): If specified, change directories before starting the process. Defaults to None.
  • as_proc (bool, default False): If True, return a subprocess.Popen object.
  • capture_output (bool, default False): If as_proc is True, capture stdout and stderr.
  • foreground (bool, default False): If True, start the subprocess as a foreground process. Defaults to False.
  • kw (Any): Additional keyword arguments to pass to meerschaum.utils.process.run_process() and by extension subprocess.Popen().
Returns
  • Either a return code integer or a subprocess.Popen object
  • (or None if a KeyboardInterrupt occurs and as_proc is True).
def attempt_import( *names: str, lazy: bool = True, warn: bool = True, install: bool = True, venv: Optional[str] = 'mrsm', precheck: bool = True, split: bool = True, check_update: bool = False, check_pypi: bool = False, check_is_installed: bool = True, color: bool = True, debug: bool = False) -> Union[Any, Tuple[Any]]:
1052def attempt_import(
1053        *names: str,
1054        lazy: bool = True,
1055        warn: bool = True,
1056        install: bool = True,
1057        venv: Optional[str] = 'mrsm',
1058        precheck: bool = True,
1059        split: bool = True,
1060        check_update: bool = False,
1061        check_pypi: bool = False,
1062        check_is_installed: bool = True,
1063        color: bool = True,
1064        debug: bool = False
1065    ) -> Union[Any, Tuple[Any]]:
1066    """
1067    Raise a warning if packages are not installed; otherwise import and return modules.
1068    If `lazy` is `True`, return lazy-imported modules.
1069    
1070    Returns tuple of modules if multiple names are provided, else returns one module.
1071    
1072    Parameters
1073    ----------
1074    names: List[str]
1075        The packages to be imported.
1076
1077    lazy: bool, default True
1078        If `True`, lazily load packages.
1079
1080    warn: bool, default True
1081        If `True`, raise a warning if a package cannot be imported.
1082
1083    install: bool, default True
1084        If `True`, attempt to install a missing package into the designated virtual environment.
1085        If `check_update` is True, install updates if available.
1086
1087    venv: Optional[str], default 'mrsm'
1088        The virtual environment in which to search for packages and to install packages into.
1089
1090    precheck: bool, default True
1091        If `True`, attempt to find module before importing (necessary for checking if modules exist
1092        and retaining lazy imports), otherwise assume lazy is `False`.
1093
1094    split: bool, default True
1095        If `True`, split packages' names on `'.'`.
1096
1097    check_update: bool, default False
1098        If `True` and `install` is `True`, install updates if the required minimum version
1099        does not match.
1100
1101    check_pypi: bool, default False
1102        If `True` and `check_update` is `True`, check PyPI when determining whether
1103        an update is required.
1104
1105    check_is_installed: bool, default True
1106        If `True`, check if the package is contained in the virtual environment.
1107
1108    Returns
1109    -------
1110    The specified modules. If they're not available and `install` is `True`, it will first
1111    download them into a virtual environment and return the modules.
1112
1113    Examples
1114    --------
1115    >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
1116    >>> pandas = attempt_import('pandas')
1117
1118    """
1119
1120    import importlib.util
1121
1122    ### to prevent recursion, check if parent Meerschaum package is being imported
1123    if names == ('meerschaum',):
1124        return _import_module('meerschaum')
1125
1126    if venv == 'mrsm' and _import_hook_venv is not None:
1127        if debug:
1128            print(f"Import hook for virtual environment '{_import_hook_venv}' is active.")
1129        venv = _import_hook_venv
1130
1131    _warnings = _import_module('meerschaum.utils.warnings')
1132    warn_function = _warnings.warn
1133
1134    def do_import(_name: str, **kw) -> Union['ModuleType', None]:
1135        with Venv(venv=venv, debug=debug):
1136            ### determine the import method (lazy vs normal)
1137            from meerschaum.utils.misc import filter_keywords
1138            import_method = (
1139                _import_module if not lazy
1140                else lazy_import
1141            )
1142            try:
1143                mod = import_method(_name, **(filter_keywords(import_method, **kw)))
1144            except Exception as e:
1145                if warn:
1146                    import traceback
1147                    traceback.print_exception(type(e), e, e.__traceback__)
1148                    warn_function(
1149                        f"Failed to import module '{_name}'.\nException:\n{e}",
1150                        ImportWarning,
1151                        stacklevel = (5 if lazy else 4),
1152                        color = False,
1153                    )
1154                mod = None
1155        return mod
1156
1157    modules = []
1158    for name in names:
1159        ### Check if package is a declared dependency.
1160        root_name = name.split('.')[0] if split else name
1161        install_name = _import_to_install_name(root_name)
1162
1163        if install_name is None:
1164            install_name = root_name
1165            if warn and root_name != 'plugins':
1166                warn_function(
1167                    f"Package '{root_name}' is not declared in meerschaum.utils.packages.",
1168                    ImportWarning,
1169                    stacklevel = 3,
1170                    color = False
1171                )
1172
1173        ### Determine if the package exists.
1174        if precheck is False:
1175            found_module = (
1176                do_import(
1177                    name, debug=debug, warn=False, venv=venv, color=color,
1178                    check_update=False, check_pypi=False, split=split,
1179                ) is not None
1180            )
1181        else:
1182            if check_is_installed:
1183                with _locks['_is_installed_first_check']:
1184                    if not _is_installed_first_check.get(name, False):
1185                        package_is_installed = is_installed(
1186                            name,
1187                            venv = venv,
1188                            split = split,
1189                            debug = debug,
1190                        )
1191                        _is_installed_first_check[name] = package_is_installed
1192                    else:
1193                        package_is_installed = _is_installed_first_check[name]
1194            else:
1195                package_is_installed = _is_installed_first_check.get(
1196                    name,
1197                    venv_contains_package(name, venv=venv, split=split, debug=debug)
1198                )
1199            found_module = package_is_installed
1200
1201        if not found_module:
1202            if install:
1203                if not pip_install(
1204                    install_name,
1205                    venv = venv,
1206                    split = False,
1207                    check_update = check_update,
1208                    color = color,
1209                    debug = debug
1210                ) and warn:
1211                    warn_function(
1212                        f"Failed to install '{install_name}'.",
1213                        ImportWarning,
1214                        stacklevel = 3,
1215                        color = False,
1216                    )
1217            elif warn:
1218                ### Raise a warning if we can't find the package and install = False.
1219                warn_function(
1220                    (f"\n\nMissing package '{name}' from virtual environment '{venv}'; "
1221                     + "some features will not work correctly."
1222                     + f"\n\nSet install=True when calling attempt_import.\n"),
1223                    ImportWarning,
1224                    stacklevel = 3,
1225                    color = False,
1226                )
1227
1228        ### Do the import. Will be lazy if lazy=True.
1229        m = do_import(
1230            name, debug=debug, warn=warn, venv=venv, color=color,
1231            check_update=check_update, check_pypi=check_pypi, install=install, split=split,
1232        )
1233        modules.append(m)
1234
1235    modules = tuple(modules)
1236    if len(modules) == 1:
1237        return modules[0]
1238    return modules

Raise a warning if packages are not installed; otherwise import and return modules. If lazy is True, return lazy-imported modules.

Returns tuple of modules if multiple names are provided, else returns one module.

Parameters
  • names (List[str]): The packages to be imported.
  • lazy (bool, default True): If True, lazily load packages.
  • warn (bool, default True): If True, raise a warning if a package cannot be imported.
  • install (bool, default True): If True, attempt to install a missing package into the designated virtual environment. If check_update is True, install updates if available.
  • venv (Optional[str], default 'mrsm'): The virtual environment in which to search for packages and to install packages into.
  • precheck (bool, default True): If True, attempt to find module before importing (necessary for checking if modules exist and retaining lazy imports), otherwise assume lazy is False.
  • split (bool, default True): If True, split packages' names on '.'.
  • check_update (bool, default False): If True and install is True, install updates if the required minimum version does not match.
  • check_pypi (bool, default False): If True and check_update is True, check PyPI when determining whether an update is required.
  • check_is_installed (bool, default True): If True, check if the package is contained in the virtual environment.
Returns
  • The specified modules. If they're not available and install is True, it will first
  • download them into a virtual environment and return the modules.
Examples
>>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
>>> pandas = attempt_import('pandas')
def lazy_import( name: str, local_name: str = None, **kw) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1241def lazy_import(
1242        name: str,
1243        local_name: str = None,
1244        **kw
1245    ) -> meerschaum.utils.packages.lazy_loader.LazyLoader:
1246    """
1247    Lazily import a package.
1248    """
1249    from meerschaum.utils.packages.lazy_loader import LazyLoader
1250    if local_name is None:
1251        local_name = name
1252    return LazyLoader(
1253        local_name,
1254        globals(),
1255        name,
1256        **kw
1257    )

Lazily import a package.

def pandas_name() -> str:
1260def pandas_name() -> str:
1261    """
1262    Return the configured name for `pandas`.
1263    
1264    Below are the expected possible values:
1265
1266    - 'pandas'
1267    - 'modin.pandas'
1268    - 'dask.dataframe'
1269
1270    """
1271    from meerschaum.config import get_config
1272    pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True)
1273    if pandas_module_name == 'modin':
1274        pandas_module_name = 'modin.pandas'
1275    elif pandas_module_name == 'dask':
1276        pandas_module_name = 'dask.dataframe'
1277
1278    return pandas_module_name

Return the configured name for pandas.

Below are the expected possible values:

  • 'pandas'
  • 'modin.pandas'
  • 'dask.dataframe'
emitted_pandas_warning: bool = False
def import_pandas(debug: bool = False, lazy: bool = False, **kw) -> "'ModuleType'":
1282def import_pandas(
1283        debug: bool = False,
1284        lazy: bool = False,
1285        **kw
1286    ) -> 'ModuleType':
1287    """
1288    Quality-of-life function to attempt to import the configured version of `pandas`.
1289    """
1290    import sys
1291    pandas_module_name = pandas_name()
1292    global emitted_pandas_warning
1293
1294    if pandas_module_name != 'pandas':
1295        with _locks['emitted_pandas_warning']:
1296            if not emitted_pandas_warning:
1297                from meerschaum.utils.warnings import warn
1298                emitted_pandas_warning = True
1299                warn(
1300                    (
1301                        "You are using an alternative Pandas implementation "
1302                        + f"'{pandas_module_name}'"
1303                        + "\n   Features may not work as expected."
1304                    ),
1305                    stack = False,
1306                )
1307
1308    pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
1309    pandas = attempt_import('pandas', debug=debug, lazy=False, **kw)
1310    pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
1311    return pd

Quality-of-life function to attempt to import the configured version of pandas.

def import_rich(lazy: bool = True, debug: bool = False, **kw: Any) -> "'ModuleType'":
1314def import_rich(
1315        lazy: bool = True,
1316        debug: bool = False,
1317        **kw : Any
1318    ) -> 'ModuleType':
1319    """
1320    Quality of life function for importing `rich`.
1321    """
1322    from meerschaum.utils.formatting import ANSI, UNICODE
1323    if not ANSI and not UNICODE:
1324        return None
1325
1326    ## need typing_extensions for `from rich import box`
1327    typing_extensions = attempt_import(
1328        'typing_extensions', lazy=False, debug=debug
1329    )
1330    pygments = attempt_import(
1331        'pygments', lazy=False,
1332    )
1333    rich = attempt_import(
1334        'rich', lazy=lazy, **kw)
1335    return rich

Quality of life function for importing rich.

def import_dcc(warn=False, **kw) -> "'ModuleType'":
1349def import_dcc(warn=False, **kw) -> 'ModuleType':
1350    """
1351    Import Dash Core Components (`dcc`).
1352    """
1353    return (
1354        attempt_import('dash_core_components', warn=warn, **kw)
1355        if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw)
1356    )

Import Dash Core Components (dcc).

def import_html(warn=False, **kw) -> "'ModuleType'":
1359def import_html(warn=False, **kw) -> 'ModuleType':
1360    """
1361    Import Dash HTML Components (`html`).
1362    """
1363    return (
1364        attempt_import('dash_html_components', warn=warn, **kw)
1365        if _dash_less_than_2(warn=warn, **kw)
1366        else attempt_import('dash.html', warn=warn, **kw)
1367    )

Import Dash HTML Components (html).

def get_modules_from_package( package: "'package'", names: bool = False, recursive: bool = False, lazy: bool = False, modules_venvs: bool = False, debug: bool = False):
1370def get_modules_from_package(
1371        package: 'package',
1372        names: bool = False,
1373        recursive: bool = False,
1374        lazy: bool = False,
1375        modules_venvs: bool = False,
1376        debug: bool = False
1377    ):
1378    """
1379    Find and import all modules in a package.
1380    
1381    Returns
1382    -------
1383    Either list of modules or tuple of lists.
1384    """
1385    from os.path import dirname, join, isfile, isdir, basename
1386    import glob
1387
1388    pattern = '*' if recursive else '*.py'
1389    package_path = dirname(package.__file__ or package.__path__[0])
1390    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
1391    _all = [
1392        basename(f)[:-3] if isfile(f) else basename(f)
1393        for f in module_names
1394            if ((isfile(f) and f.endswith('.py')) or isdir(f))
1395               and not f.endswith('__init__.py')
1396               and not f.endswith('__pycache__')
1397    ]
1398
1399    if debug:
1400        from meerschaum.utils.debug import dprint
1401        dprint(str(_all))
1402    modules = []
1403    for module_name in [package.__name__ + "." + mod_name for mod_name in _all]:
1404        ### there's probably a better way than a try: catch but it'll do for now
1405        try:
1406            ### if specified, activate the module's virtual environment before importing.
1407            ### NOTE: this only considers the filename, so two modules from different packages
1408            ### may end up sharing virtual environments.
1409            if modules_venvs:
1410                activate_venv(module_name.split('.')[-1], debug=debug)
1411            m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name)
1412            modules.append(m)
1413        except Exception as e:
1414            if debug:
1415                dprint(str(e))
1416        finally:
1417            if modules_venvs:
1418                deactivate_venv(module_name.split('.')[-1], debug=debug)
1419    if names:
1420        return _all, modules
1421
1422    return modules

Find and import all modules in a package.

Returns
  • Either list of modules or tuple of lists.
def import_children( package: "Optional['ModuleType']" = None, package_name: Optional[str] = None, types: Optional[List[str]] = None, lazy: bool = True, recursive: bool = False, debug: bool = False) -> "List['ModuleType']":
1425def import_children(
1426        package: Optional['ModuleType'] = None,
1427        package_name: Optional[str] = None,
1428        types : Optional[List[str]] = None,
1429        lazy: bool = True,
1430        recursive: bool = False,
1431        debug: bool = False
1432    ) -> List['ModuleType']:
1433    """
1434    Import all functions in a package to its `__init__`.
1435
1436    Parameters
1437    ----------
1438    package: Optional[ModuleType], default None
1439        Package to import its functions into.
1440        If `None` (default), use parent.
1441
1442    package_name: Optional[str], default None
1443        Name of package to import its functions into
1444        If None (default), use parent.
1445
1446    types: Optional[List[str]], default None
1447        Types of members to return.
1448        Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']`
1449
1450    Returns
1451    -------
1452    A list of modules.
1453    """
1454    import sys, inspect
1455
1456    if types is None:
1457        types = ['method', 'builtin', 'function', 'class', 'module']
1458
1459    ### if package_name and package are None, use parent
1460    if package is None and package_name is None:
1461        package_name = inspect.stack()[1][0].f_globals['__name__']
1462
1463    ### populate package or package_name from other other
1464    if package is None:
1465        package = sys.modules[package_name]
1466    elif package_name is None:
1467        package_name = package.__name__
1468
1469    ### Set attributes in sys module version of package.
1470    ### Kinda like setting a dictionary
1471    ###   functions[name] = func
1472    modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug)
1473    _all, members = [], []
1474    objects = []
1475    for module in modules:
1476        _objects = []
1477        for ob in inspect.getmembers(module):
1478            for t in types:
1479                ### ob is a tuple of (name, object)
1480                if getattr(inspect, 'is' + t)(ob[1]):
1481                    _objects.append(ob)
1482
1483        if 'module' in types:
1484            _objects.append((module.__name__.split('.')[0], module))
1485        objects += _objects
1486    for ob in objects:
1487        setattr(sys.modules[package_name], ob[0], ob[1])
1488        _all.append(ob[0])
1489        members.append(ob[1])
1490
1491    if debug:
1492        from meerschaum.utils.debug import dprint
1493        dprint(str(_all))
1494    ### set __all__ for import *
1495    setattr(sys.modules[package_name], '__all__', _all)
1496    return members

Import all functions in a package to its __init__.

Parameters
  • package (Optional[ModuleType], default None): Package to import its functions into. If None (default), use parent.
  • package_name (Optional[str], default None): Name of package to import its functions into If None (default), use parent.
  • types (Optional[List[str]], default None): Types of members to return. Defaults are ['method', 'builtin', 'class', 'function', 'package', 'module']
Returns
  • A list of modules.
def reload_package( package: str, skip_submodules: Optional[List[str]] = None, lazy: bool = False, debug: bool = False, **kw: Any):
1500def reload_package(
1501        package: str,
1502        skip_submodules: Optional[List[str]] = None,
1503        lazy: bool = False,
1504        debug: bool = False,
1505        **kw: Any
1506    ):
1507    """
1508    Recursively load a package's subpackages, even if they were not previously loaded.
1509    """
1510    import sys
1511    if isinstance(package, str):
1512        package_name = package
1513    else:
1514        try:
1515            package_name = package.__name__
1516        except Exception as e:
1517            package_name = str(package)
1518
1519    skip_submodules = skip_submodules or []
1520    if 'meerschaum.utils.packages' not in skip_submodules:
1521        skip_submodules.append('meerschaum.utils.packages')
1522    def safeimport():
1523        subs = [
1524            m for m in sys.modules
1525            if m.startswith(package_name + '.')
1526        ]
1527        subs_to_skip = []
1528        for skip_mod in skip_submodules:
1529            for mod in subs:
1530                if mod.startswith(skip_mod):
1531                    subs_to_skip.append(mod)
1532                    continue
1533
1534        subs = [m for m in subs if m not in subs_to_skip]
1535        for module_name in subs:
1536            _reload_module_cache[module_name] = sys.modules.pop(module_name, None)
1537        if not subs_to_skip:
1538            _reload_module_cache[package_name] = sys.modules.pop(package_name, None)
1539
1540        return _import_module(package_name)
1541
1542    return safeimport()

Recursively load a package's subpackages, even if they were not previously loaded.

def reload_meerschaum(debug: bool = False) -> Tuple[bool, str]:
1545def reload_meerschaum(debug: bool = False) -> SuccessTuple:
1546    """
1547    Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1548    """
1549    reload_package(
1550        'meerschaum',
1551        skip_submodules = [
1552            'meerschaum._internal.shell',
1553            'meerschaum.utils.pool',
1554        ]
1555    )
1556
1557    from meerschaum.plugins import reload_plugins
1558    from meerschaum._internal.shell.Shell import _insert_shell_actions
1559    reload_plugins(debug=debug)
1560    _insert_shell_actions()
1561    return True, "Success"

Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.

def is_installed( import_name: str, venv: Optional[str] = 'mrsm', split: bool = True, debug: bool = False) -> bool:
1564def is_installed(
1565        import_name: str,
1566        venv: Optional[str] = 'mrsm',
1567        split: bool = True,
1568        debug: bool = False,
1569    ) -> bool:
1570    """
1571    Check whether a package is installed.
1572    """
1573    if debug:
1574        from meerschaum.utils.debug import dprint
1575    root_name = import_name.split('.')[0] if split else import_name
1576    import importlib.util
1577    with Venv(venv, debug=debug):
1578        try:
1579            spec_path = pathlib.Path(
1580                get_module_path(root_name, venv=venv, debug=debug)
1581                or
1582                importlib.util.find_spec(root_name).origin
1583            )
1584        except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e:
1585            spec_path = None
1586
1587        found = (
1588            not need_update(
1589                None, import_name = root_name,
1590                _run_determine_version = False,
1591                check_pypi = False,
1592                version = determine_version(
1593                    spec_path, venv=venv, debug=debug, import_name=root_name
1594                ),
1595                debug = debug,
1596            )
1597        ) if spec_path is not None else False
1598
1599    return found

Check whether a package is installed.

def venv_contains_package( import_name: str, venv: Optional[str] = 'mrsm', split: bool = True, debug: bool = False) -> bool:
1602def venv_contains_package(
1603        import_name: str,
1604        venv: Optional[str] = 'mrsm',
1605        split: bool = True,
1606        debug: bool = False,
1607    ) -> bool:
1608    """
1609    Search the contents of a virtual environment for a package.
1610    """
1611    root_name = import_name.split('.')[0] if split else import_name
1612    return get_module_path(root_name, venv=venv, debug=debug) is not None

Search the contents of a virtual environment for a package.

def package_venv(package: "'ModuleType'") -> Optional[str]:
1615def package_venv(package: 'ModuleType') -> Union[str, None]:
1616    """
1617    Inspect a package and return the virtual environment in which it presides.
1618    """
1619    import os
1620    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
1621    if str(VIRTENV_RESOURCES_PATH) not in package.__file__:
1622        return None
1623    return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]

Inspect a package and return the virtual environment in which it presides.

def ensure_readline() -> "'ModuleType'":
1626def ensure_readline() -> 'ModuleType':
1627    """Make sure that the `readline` package is able to be imported."""
1628    import sys
1629    try:
1630        import readline
1631    except ImportError:
1632        readline = None
1633
1634    if readline is None:
1635        import platform
1636        rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3"
1637        try:
1638            rl = attempt_import(
1639                rl_name,
1640                lazy = False,
1641                install = True,
1642                venv = None,
1643                warn = False,
1644            )
1645        except (ImportError, ModuleNotFoundError):
1646            if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None):
1647                print(f"Unable to import {rl_name}!", file=sys.stderr)
1648                sys.exit(1)
1649
1650    sys.modules['readline'] = readline
1651    return readline

Make sure that the readline package is able to be imported.