meerschaum.utils.packages
Functions for managing packages and virtual environments reside here.
1#! /usr/bin/env python 2# -*- coding: utf-8 -*- 3# vim:fenc=utf-8 4 5""" 6Functions for managing packages and virtual environments reside here. 7""" 8 9from __future__ import annotations 10 11import importlib.util, os, pathlib, re 12from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable 13from meerschaum.utils.threading import Lock, RLock 14from meerschaum.utils.packages._packages import packages, all_packages, get_install_names 15from meerschaum.utils.venv import ( 16 activate_venv, 17 deactivate_venv, 18 venv_executable, 19 venv_exec, 20 venv_exists, 21 venv_target_path, 22 inside_venv, 23 Venv, 24 init_venv, 25) 26 27_import_module = importlib.import_module 28_import_hook_venv = None 29_locks = { 30 '_pkg_resources_get_distribution': RLock(), 31 'import_versions': RLock(), 32 '_checked_for_updates': RLock(), 33 '_is_installed_first_check': RLock(), 34 'emitted_pandas_warning': RLock(), 35} 36_checked_for_updates = set() 37_is_installed_first_check: Dict[str, bool] = {} 38_MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/" 39 40def get_module_path( 41 import_name: str, 42 venv: Optional[str] = 'mrsm', 43 debug: bool = False, 44 _try_install_name_on_fail: bool = True, 45 ) -> Union[pathlib.Path, None]: 46 """ 47 Get a module's path without importing. 48 """ 49 import site 50 if debug: 51 from meerschaum.utils.debug import dprint 52 if not _try_install_name_on_fail: 53 install_name = _import_to_install_name(import_name, with_version=False) 54 install_name_lower = install_name.lower().replace('-', '_') 55 import_name_lower = install_name_lower 56 else: 57 import_name_lower = import_name.lower().replace('-', '_') 58 59 vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug) 60 if not vtp.exists(): 61 if debug: 62 dprint( 63 ( 64 "Venv '{venv}' does not exist, cannot import " 65 + f"'{import_name}'." 66 ), 67 color = False, 68 ) 69 return None 70 71 venv_target_candidate_paths = [vtp] 72 if venv is None: 73 site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())] 74 site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()] 75 76 paths_to_add = [ 77 path 78 for path in site_user_packages_dirs + site_packages_dirs 79 if path not in venv_target_candidate_paths 80 ] 81 venv_target_candidate_paths += paths_to_add 82 83 candidates = [] 84 for venv_target_candidate in venv_target_candidate_paths: 85 try: 86 file_names = os.listdir(venv_target_candidate) 87 except FileNotFoundError: 88 continue 89 for file_name in file_names: 90 file_name_lower = file_name.lower().replace('-', '_') 91 if not file_name_lower.startswith(import_name_lower): 92 continue 93 if file_name.endswith('dist_info'): 94 continue 95 file_path = venv_target_candidate / file_name 96 97 ### Most likely: Is a directory with __init__.py 98 if file_name_lower == import_name_lower and file_path.is_dir(): 99 init_path = file_path / '__init__.py' 100 if init_path.exists(): 101 candidates.append(init_path) 102 103 ### May be a standalone .py file. 104 elif file_name_lower == import_name_lower + '.py': 105 candidates.append(file_path) 106 107 ### Compiled wheels (e.g. pyodbc) 108 elif file_name_lower.startswith(import_name_lower + '.'): 109 candidates.append(file_path) 110 111 if len(candidates) == 1: 112 return candidates[0] 113 114 if not candidates: 115 if _try_install_name_on_fail: 116 return get_module_path( 117 import_name, venv=venv, debug=debug, 118 _try_install_name_on_fail=False 119 ) 120 return None 121 122 specs_paths = [] 123 for candidate_path in candidates: 124 spec = importlib.util.spec_from_file_location(import_name, str(candidate_path)) 125 if spec is not None: 126 return candidate_path 127 128 return None 129 130 131def manually_import_module( 132 import_name: str, 133 venv: Optional[str] = 'mrsm', 134 check_update: bool = True, 135 check_pypi: bool = False, 136 install: bool = True, 137 split: bool = True, 138 warn: bool = True, 139 color: bool = True, 140 debug: bool = False, 141 use_sys_modules: bool = True, 142 ) -> Union['ModuleType', None]: 143 """ 144 Manually import a module from a virtual environment (or the base environment). 145 146 Parameters 147 ---------- 148 import_name: str 149 The name of the module. 150 151 venv: Optional[str], default 'mrsm' 152 The virtual environment to read from. 153 154 check_update: bool, default True 155 If `True`, examine whether the available version of the package meets the required version. 156 157 check_pypi: bool, default False 158 If `True`, check PyPI for updates before importing. 159 160 install: bool, default True 161 If `True`, install the package if it's not installed or needs an update. 162 163 split: bool, default True 164 If `True`, split `import_name` on periods to get the package name. 165 166 warn: bool, default True 167 If `True`, raise a warning if the package cannot be imported. 168 169 color: bool, default True 170 If `True`, use color output for debug and warning text. 171 172 debug: bool, default False 173 Verbosity toggle. 174 175 use_sys_modules: bool, default True 176 If `True`, return the module in `sys.modules` if it exists. 177 Otherwise continue with manually importing. 178 179 Returns 180 ------- 181 The specified module or `None` if it can't be imported. 182 183 """ 184 import sys 185 _previously_imported = import_name in sys.modules 186 if _previously_imported and use_sys_modules: 187 return sys.modules[import_name] 188 if debug: 189 from meerschaum.utils.debug import dprint 190 from meerschaum.utils.warnings import warn as warn_function 191 import warnings 192 root_name = import_name.split('.')[0] if split else import_name 193 install_name = _import_to_install_name(root_name) 194 195 root_path = get_module_path(root_name, venv=venv) 196 if root_path is None: 197 return None 198 mod_path = root_path 199 if mod_path.is_dir(): 200 for _dir in import_name.split('.')[:-1]: 201 mod_path = mod_path / _dir 202 possible_end_module_filename = import_name.split('.')[-1] + '.py' 203 try: 204 mod_path = ( 205 (mod_path / possible_end_module_filename) 206 if possible_end_module_filename in os.listdir(mod_path) 207 else ( 208 mod_path / import_name.split('.')[-1] / '__init__.py' 209 ) 210 ) 211 except Exception as e: 212 mod_path = None 213 214 spec = ( 215 importlib.util.find_spec(import_name) if mod_path is None or not mod_path.exists() 216 else importlib.util.spec_from_file_location(import_name, str(mod_path)) 217 ) 218 root_spec = ( 219 importlib.util.find_spec(root_name) if not root_path.exists() 220 else importlib.util.spec_from_file_location(root_name, str(root_path)) 221 ) 222 223 ### Check for updates before importing. 224 _version = ( 225 determine_version( 226 pathlib.Path(root_spec.origin), 227 import_name=root_name, venv=venv, debug=debug 228 ) if root_spec is not None and root_spec.origin is not None else None 229 ) 230 231 if _version is not None: 232 if check_update: 233 if need_update( 234 None, 235 import_name = root_name, 236 version = _version, 237 check_pypi = check_pypi, 238 debug = debug, 239 ): 240 if install: 241 if not pip_install( 242 root_name, 243 venv = venv, 244 split = False, 245 check_update = check_update, 246 color = color, 247 debug = debug 248 ) and warn: 249 warn_function( 250 f"There's an update available for '{install_name}', " 251 + "but it failed to install. " 252 + "Try installig via Meerschaum with " 253 + "`install packages '{install_name}'`.", 254 ImportWarning, 255 stacklevel = 3, 256 color = False, 257 ) 258 elif warn: 259 warn_function( 260 f"There's an update available for '{root_name}'.", 261 stack = False, 262 color = False, 263 ) 264 spec = ( 265 importlib.util.find_spec(import_name) 266 if mod_path is None or not mod_path.exists() 267 else importlib.util.spec_from_file_location(import_name, str(mod_path)) 268 ) 269 270 271 if spec is None: 272 try: 273 mod = _import_module(import_name) 274 except Exception as e: 275 mod = None 276 return mod 277 278 with Venv(venv, debug=debug): 279 mod = importlib.util.module_from_spec(spec) 280 old_sys_mod = sys.modules.get(import_name, None) 281 sys.modules[import_name] = mod 282 283 try: 284 with warnings.catch_warnings(): 285 warnings.filterwarnings('ignore', 'The NumPy') 286 spec.loader.exec_module(mod) 287 except Exception as e: 288 pass 289 mod = _import_module(import_name) 290 if old_sys_mod is not None: 291 sys.modules[import_name] = old_sys_mod 292 else: 293 del sys.modules[import_name] 294 295 return mod 296 297 298def _import_to_install_name(import_name: str, with_version: bool = True) -> str: 299 """ 300 Try to translate an import name to an installation name. 301 """ 302 install_name = all_packages.get(import_name, import_name) 303 if with_version: 304 return install_name 305 return get_install_no_version(install_name) 306 307 308def _import_to_dir_name(import_name: str) -> str: 309 """ 310 Translate an import name to the package name in the sites-packages directory. 311 """ 312 import re 313 return re.split( 314 r'[<>=\[]', all_packages.get(import_name, import_name) 315 )[0].replace('-', '_').lower() 316 317 318def _install_to_import_name(install_name: str) -> str: 319 """ 320 Translate an installation name to a package's import name. 321 """ 322 _install_no_version = get_install_no_version(install_name) 323 return get_install_names().get(_install_no_version, _install_no_version) 324 325 326def get_install_no_version(install_name: str) -> str: 327 """ 328 Strip the version information from the install name. 329 """ 330 import re 331 return re.split(r'[\[=<>,! \]]', install_name)[0] 332 333 334import_versions = {} 335def determine_version( 336 path: pathlib.Path, 337 import_name: Optional[str] = None, 338 venv: Optional[str] = 'mrsm', 339 search_for_metadata: bool = True, 340 split: bool = True, 341 warn: bool = False, 342 debug: bool = False, 343 ) -> Union[str, None]: 344 """ 345 Determine a module's `__version__` string from its filepath. 346 347 First it searches for pip metadata, then it attempts to import the module in a subprocess. 348 349 Parameters 350 ---------- 351 path: pathlib.Path 352 The file path of the module. 353 354 import_name: Optional[str], default None 355 The name of the module. If omitted, it will be determined from the file path. 356 Defaults to `None`. 357 358 venv: Optional[str], default 'mrsm' 359 The virtual environment of the Python interpreter to use if importing is necessary. 360 361 search_for_metadata: bool, default True 362 If `True`, search the pip site_packages directory (assumed to be the parent) 363 for the corresponding dist-info directory. 364 365 warn: bool, default True 366 If `True`, raise a warning if the module fails to import in the subprocess. 367 368 split: bool, default True 369 If `True`, split the determined import name by periods to get the room name. 370 371 Returns 372 ------- 373 The package's version string if available or `None`. 374 If multiple versions are found, it will trigger an import in a subprocess. 375 376 """ 377 with _locks['import_versions']: 378 if venv not in import_versions: 379 import_versions[venv] = {} 380 import importlib.metadata 381 import re, os 382 old_cwd = os.getcwd() 383 if debug: 384 from meerschaum.utils.debug import dprint 385 from meerschaum.utils.warnings import warn as warn_function 386 if import_name is None: 387 import_name = path.parent.stem if path.stem == '__init__' else path.stem 388 import_name = import_name.split('.')[0] if split else import_name 389 if import_name in import_versions[venv]: 390 return import_versions[venv][import_name] 391 _version = None 392 module_parent_dir = ( 393 path.parent.parent if path.stem == '__init__' else path.parent 394 ) if path is not None else venv_target_path(venv, debug=debug) 395 396 installed_dir_name = _import_to_dir_name(import_name) 397 clean_installed_dir_name = installed_dir_name.lower().replace('-', '_') 398 399 ### First, check if a dist-info directory exists. 400 _found_versions = [] 401 if search_for_metadata: 402 for filename in os.listdir(module_parent_dir): 403 if not filename.endswith('.dist-info'): 404 continue 405 filename_lower = filename.lower() 406 if not filename_lower.startswith(clean_installed_dir_name + '-'): 407 continue 408 _v = filename.replace('.dist-info', '').split("-")[-1] 409 _found_versions.append(_v) 410 411 if len(_found_versions) == 1: 412 _version = _found_versions[0] 413 with _locks['import_versions']: 414 import_versions[venv][import_name] = _version 415 return _found_versions[0] 416 417 if not _found_versions: 418 try: 419 import importlib.metadata as importlib_metadata 420 except ImportError: 421 importlib_metadata = attempt_import( 422 'importlib_metadata', 423 debug=debug, check_update=False, precheck=False, 424 color=False, check_is_installed=False, lazy=False, 425 ) 426 try: 427 os.chdir(module_parent_dir) 428 _version = importlib_metadata.metadata(import_name)['Version'] 429 except Exception as e: 430 _version = None 431 finally: 432 os.chdir(old_cwd) 433 434 if _version is not None: 435 with _locks['import_versions']: 436 import_versions[venv][import_name] = _version 437 return _version 438 439 if debug: 440 print(f'Found multiple versions for {import_name}: {_found_versions}') 441 442 module_parent_dir_str = module_parent_dir.as_posix() 443 444 ### Not a pip package, so let's try importing the module directly (in a subprocess). 445 _no_version_str = 'no-version' 446 code = ( 447 f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n" 448 + f"module = importlib.import_module('{import_name}');\n" 449 + "try:\n" 450 + " print(module.__version__ , end='')\n" 451 + "except:\n" 452 + f" print('{_no_version_str}', end='')" 453 ) 454 exit_code, stdout_bytes, stderr_bytes = venv_exec( 455 code, venv=venv, with_extras=True, debug=debug 456 ) 457 stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8') 458 _version = stdout.split('\n')[-1] if exit_code == 0 else None 459 _version = _version if _version != _no_version_str else None 460 461 if _version is None: 462 _version = _get_package_metadata(import_name, venv).get('version', None) 463 if _version is None and warn: 464 warn_function( 465 f"Failed to determine a version for '{import_name}':\n{stderr}", 466 stack = False 467 ) 468 469 ### If `__version__` doesn't exist, return `None`. 470 import_versions[venv][import_name] = _version 471 return _version 472 473 474def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, str]: 475 """ 476 Get a package's metadata from pip. 477 This is useful for getting a version when no `__version__` is defined 478 and multiple versions are installed. 479 480 Parameters 481 ---------- 482 import_name: str 483 The package's import or installation name. 484 485 venv: Optional[str] 486 The virtual environment which contains the package. 487 488 Returns 489 ------- 490 A dictionary of metadata from pip. 491 """ 492 import re 493 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 494 install_name = _import_to_install_name(import_name) 495 _args = ['pip', 'show', install_name] 496 if venv is not None: 497 cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache' 498 _args += ['--cache-dir', cache_dir_path.as_posix()] 499 500 proc = run_python_package( 501 'uv', _args, 502 capture_output=True, as_proc=True, venv=venv, universal_newlines=True, 503 ) 504 outs, errs = proc.communicate() 505 lines = outs.split('\n') 506 meta = {} 507 for line in lines: 508 vals = line.split(": ") 509 if len(vals) != 2: 510 continue 511 k, v = vals[0].lower(), vals[1] 512 if v and 'UNKNOWN' not in v: 513 meta[k] = v 514 return meta 515 516 517def need_update( 518 package: Optional['ModuleType'] = None, 519 install_name: Optional[str] = None, 520 import_name: Optional[str] = None, 521 version: Optional[str] = None, 522 check_pypi: bool = False, 523 split: bool = True, 524 color: bool = True, 525 debug: bool = False, 526 _run_determine_version: bool = True, 527 ) -> bool: 528 """ 529 Check if a Meerschaum dependency needs an update. 530 Returns a bool for whether or not a package needs to be updated. 531 532 Parameters 533 ---------- 534 package: 'ModuleType' 535 The module of the package to be updated. 536 537 install_name: Optional[str], default None 538 If provided, use this string to determine the required version. 539 Otherwise use the install name defined in `meerschaum.utils.packages._packages`. 540 541 import_name: 542 If provided, override the package's `__name__` string. 543 544 version: Optional[str], default None 545 If specified, override the package's `__version__` string. 546 547 check_pypi: bool, default False 548 If `True`, check pypi.org for updates. 549 Defaults to `False`. 550 551 split: bool, default True 552 If `True`, split the module's name on periods to detrive the root name. 553 Defaults to `True`. 554 555 color: bool, default True 556 If `True`, format debug output. 557 Defaults to `True`. 558 559 debug: bool, default True 560 Verbosity toggle. 561 562 Returns 563 ------- 564 A bool indicating whether the package requires an update. 565 566 """ 567 if debug: 568 from meerschaum.utils.debug import dprint 569 from meerschaum.utils.warnings import warn as warn_function 570 import re 571 root_name = ( 572 package.__name__.split('.')[0] if split else package.__name__ 573 ) if import_name is None else ( 574 import_name.split('.')[0] if split else import_name 575 ) 576 install_name = install_name or _import_to_install_name(root_name) 577 with _locks['_checked_for_updates']: 578 if install_name in _checked_for_updates: 579 return False 580 _checked_for_updates.add(install_name) 581 582 _install_no_version = get_install_no_version(install_name) 583 required_version = install_name.replace(_install_no_version, '') 584 if ']' in required_version: 585 required_version = required_version.split(']')[1] 586 587 ### No minimum version was specified, and we're not going to check PyPI. 588 if not required_version and not check_pypi: 589 return False 590 591 ### NOTE: Sometimes (rarely), we depend on a development build of a package. 592 if '.dev' in required_version: 593 required_version = required_version.split('.dev')[0] 594 if version and '.dev' in version: 595 version = version.split('.dev')[0] 596 597 try: 598 if not version: 599 if not _run_determine_version: 600 version = determine_version( 601 pathlib.Path(package.__file__), 602 import_name=root_name, warn=False, debug=debug 603 ) 604 if version is None: 605 return False 606 except Exception as e: 607 if debug: 608 dprint(str(e), color=color) 609 dprint("No version could be determined from the installed package.", color=color) 610 return False 611 split_version = version.split('.') 612 last_part = split_version[-1] 613 if len(split_version) == 2: 614 version = '.'.join(split_version) + '.0' 615 elif 'dev' in last_part or 'rc' in last_part: 616 tag = 'dev' if 'dev' in last_part else 'rc' 617 last_sep = '-' 618 if not last_part.startswith(tag): 619 last_part = f'-{tag}'.join(last_part.split(tag)) 620 last_sep = '.' 621 version = '.'.join(split_version[:-1]) + last_sep + last_part 622 elif len(split_version) > 3: 623 version = '.'.join(split_version[:3]) 624 625 packaging_version = attempt_import( 626 'packaging.version', check_update=False, lazy=False, debug=debug, 627 ) 628 629 ### Get semver if necessary 630 if required_version: 631 semver_path = get_module_path('semver', debug=debug) 632 if semver_path is None: 633 pip_install(_import_to_install_name('semver'), debug=debug) 634 semver = attempt_import('semver', check_update=False, lazy=False, debug=debug) 635 if check_pypi: 636 ### Check PyPI for updates 637 update_checker = attempt_import( 638 'update_checker', lazy=False, check_update=False, debug=debug 639 ) 640 checker = update_checker.UpdateChecker() 641 result = checker.check(_install_no_version, version) 642 else: 643 ### Skip PyPI and assume we can't be sure. 644 result = None 645 646 ### Compare PyPI's version with our own. 647 if result is not None: 648 ### We have a result from PyPI and a stated required version. 649 if required_version: 650 try: 651 return semver.Version.parse(result.available_version).match(required_version) 652 except AttributeError as e: 653 pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug) 654 semver = manually_import_module('semver', venv='mrsm') 655 return semver.Version.parse(version).match(required_version) 656 except Exception as e: 657 if debug: 658 dprint(f"Failed to match versions with exception:\n{e}", color=color) 659 return False 660 661 ### If `check_pypi` and we don't have a required version, check if PyPI's version 662 ### is newer than the installed version. 663 else: 664 return ( 665 packaging_version.parse(result.available_version) > 666 packaging_version.parse(version) 667 ) 668 669 ### We might be depending on a prerelease. 670 ### Sanity check that the required version is not greater than the installed version. 671 required_version = ( 672 required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '') 673 .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '') 674 .replace('-py3-none-any.whl', '') 675 ) 676 677 if 'a' in required_version: 678 required_version = required_version.replace('a', '-dev').replace('+mrsm', '') 679 version = version.replace('a', '-dev').replace('+mrsm', '') 680 try: 681 return ( 682 (not semver.Version.parse(version).match(required_version)) 683 if required_version else False 684 ) 685 except AttributeError as e: 686 pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug) 687 semver = manually_import_module('semver', venv='mrsm', debug=debug) 688 return ( 689 (not semver.Version.parse(version).match(required_version)) 690 if required_version else False 691 ) 692 except Exception as e: 693 print(f"Unable to parse version ({version}) for package '{import_name}'.") 694 print(e) 695 if debug: 696 dprint(e) 697 return False 698 try: 699 return ( 700 packaging_version.parse(version) > 701 packaging_version.parse(required_version) 702 ) 703 except Exception as e: 704 if debug: 705 dprint(e) 706 return False 707 return False 708 709 710def get_pip( 711 venv: Optional[str] = 'mrsm', 712 color: bool = True, 713 debug: bool = False, 714 ) -> bool: 715 """ 716 Download and run the get-pip.py script. 717 718 Parameters 719 ---------- 720 venv: Optional[str], default 'mrsm' 721 The virtual environment into which to install `pip`. 722 723 color: bool, default True 724 If `True`, force color output. 725 726 debug: bool, default False 727 Verbosity toggle. 728 729 Returns 730 ------- 731 A bool indicating success. 732 733 """ 734 import sys, subprocess 735 from meerschaum.utils.misc import wget 736 from meerschaum.config._paths import CACHE_RESOURCES_PATH 737 from meerschaum.config.static import STATIC_CONFIG 738 url = STATIC_CONFIG['system']['urls']['get-pip.py'] 739 dest = CACHE_RESOURCES_PATH / 'get-pip.py' 740 try: 741 wget(url, dest, color=False, debug=debug) 742 except Exception as e: 743 print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 744 sys.exit(1) 745 if venv is not None: 746 init_venv(venv=venv, debug=debug) 747 cmd_list = [venv_executable(venv=venv), dest.as_posix()] 748 return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0 749 750 751def pip_install( 752 *install_names: str, 753 args: Optional[List[str]] = None, 754 requirements_file_path: Union[pathlib.Path, str, None] = None, 755 venv: Optional[str] = 'mrsm', 756 split: bool = False, 757 check_update: bool = True, 758 check_pypi: bool = True, 759 check_wheel: bool = True, 760 _uninstall: bool = False, 761 _from_completely_uninstall: bool = False, 762 _install_uv_pip: bool = True, 763 color: bool = True, 764 silent: bool = False, 765 debug: bool = False, 766 ) -> bool: 767 """ 768 Install packages from PyPI with `pip`. 769 770 Parameters 771 ---------- 772 *install_names: str 773 The installation names of packages to be installed. 774 This includes version restrictions. 775 Use `_import_to_install_name()` to get the predefined `install_name` for a package 776 from its import name. 777 778 args: Optional[List[str]], default None 779 A list of command line arguments to pass to `pip`. 780 If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`. 781 782 requirements_file_path: Optional[pathlib.Path, str], default None 783 If provided, append `['-r', '/path/to/requirements.txt']` to `args`. 784 785 venv: str, default 'mrsm' 786 The virtual environment to install into. 787 788 split: bool, default False 789 If `True`, split on periods and only install the root package name. 790 791 check_update: bool, default True 792 If `True`, check if the package requires an update. 793 794 check_pypi: bool, default True 795 If `True` and `check_update` is `True`, check PyPI for the latest version. 796 797 check_wheel: bool, default True 798 If `True`, check if `wheel` is available. 799 800 _uninstall: bool, default False 801 If `True`, uninstall packages instead. 802 803 color: bool, default True 804 If `True`, include color in debug text. 805 806 silent: bool, default False 807 If `True`, skip printing messages. 808 809 debug: bool, default False 810 Verbosity toggle. 811 812 Returns 813 ------- 814 A bool indicating success. 815 816 """ 817 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 818 from meerschaum.config import get_config 819 from meerschaum.config.static import STATIC_CONFIG 820 from meerschaum.utils.warnings import warn 821 from meerschaum.utils.misc import is_android 822 if args is None: 823 args = ['--upgrade'] if not _uninstall else [] 824 if color: 825 ANSI, UNICODE = True, True 826 else: 827 ANSI, UNICODE = False, False 828 if check_wheel: 829 have_wheel = venv_contains_package('wheel', venv=venv, debug=debug) 830 831 daemon_env_var = STATIC_CONFIG['environment']['daemon_id'] 832 inside_daemon = daemon_env_var in os.environ 833 if inside_daemon: 834 silent = True 835 836 _args = list(args) 837 have_pip = venv_contains_package('pip', venv=None, debug=debug) 838 try: 839 import pip 840 have_pip = True 841 except ImportError: 842 have_pip = False 843 try: 844 import uv 845 uv_bin = uv.find_uv_bin() 846 have_uv_pip = True 847 except (ImportError, FileNotFoundError): 848 uv_bin = None 849 have_uv_pip = False 850 if have_pip and not have_uv_pip and _install_uv_pip and not is_android(): 851 if not pip_install( 852 'uv', 853 venv=None, 854 debug=debug, 855 _install_uv_pip=False, 856 check_update=False, 857 check_pypi=False, 858 check_wheel=False, 859 ) and not silent: 860 warn( 861 f"Failed to install `uv` for virtual environment '{venv}'.", 862 color=False, 863 ) 864 865 use_uv_pip = ( 866 venv_contains_package('uv', venv=None, debug=debug) 867 and uv_bin is not None 868 and venv is not None 869 ) 870 871 import sys 872 if not have_pip and not use_uv_pip: 873 if not get_pip(venv=venv, color=color, debug=debug): 874 import sys 875 minor = sys.version_info.minor 876 print( 877 "\nFailed to import `pip` and `ensurepip`.\n" 878 + "If you are running Ubuntu/Debian, " 879 + "you might need to install `python3.{minor}-distutils`:\n\n" 880 + f" sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n" 881 + "Please install pip and restart Meerschaum.\n\n" 882 + "You can find instructions on installing `pip` here:\n" 883 + "https://pip.pypa.io/en/stable/installing/" 884 ) 885 sys.exit(1) 886 887 with Venv(venv, debug=debug): 888 if venv is not None: 889 if ( 890 '--ignore-installed' not in args 891 and '-I' not in _args 892 and not _uninstall 893 and not use_uv_pip 894 ): 895 _args += ['--ignore-installed'] 896 if '--cache-dir' not in args and not _uninstall: 897 cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache' 898 _args += ['--cache-dir', str(cache_dir_path)] 899 900 if 'pip' not in ' '.join(_args) and not use_uv_pip: 901 if check_update and not _uninstall: 902 pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False) 903 if need_update(pip, check_pypi=check_pypi, debug=debug): 904 _args.append(all_packages['pip']) 905 906 _args = (['install'] if not _uninstall else ['uninstall']) + _args 907 908 if check_wheel and not _uninstall and not use_uv_pip: 909 if not have_wheel: 910 setup_packages_to_install = ( 911 ['setuptools', 'wheel'] 912 + ([] if is_android() else ['uv']) 913 ) 914 if not pip_install( 915 *setup_packages_to_install, 916 venv=venv, 917 check_update=False, 918 check_pypi=False, 919 check_wheel=False, 920 debug=debug, 921 _install_uv_pip=False, 922 ) and not silent: 923 from meerschaum.utils.misc import items_str 924 warn( 925 ( 926 f"Failed to install {items_str(setup_packages_to_install)} for virtual " 927 + f"environment '{venv}'." 928 ), 929 color=False, 930 ) 931 932 if requirements_file_path is not None: 933 _args.append('-r') 934 _args.append(pathlib.Path(requirements_file_path).resolve().as_posix()) 935 936 if not ANSI and '--no-color' not in _args: 937 _args.append('--no-color') 938 939 if '--no-input' not in _args and not use_uv_pip: 940 _args.append('--no-input') 941 942 if _uninstall and '-y' not in _args and not use_uv_pip: 943 _args.append('-y') 944 945 if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip: 946 _args.append('--no-warn-conflicts') 947 948 if '--disable-pip-version-check' not in _args and not use_uv_pip: 949 _args.append('--disable-pip-version-check') 950 951 if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall): 952 if venv is not None: 953 _args += ['--target', venv_target_path(venv, debug=debug)] 954 elif ( 955 '--target' not in _args 956 and '-t' not in _args 957 and not inside_venv() 958 and not _uninstall 959 and not use_uv_pip 960 ): 961 _args += ['--user'] 962 963 if debug: 964 if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args: 965 if use_uv_pip: 966 _args.append('--verbose') 967 else: 968 if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args: 969 pass 970 971 _packages = [ 972 (install_name if not _uninstall else get_install_no_version(install_name)) 973 for install_name in install_names 974 ] 975 msg = "Installing packages:" if not _uninstall else "Uninstalling packages:" 976 for p in _packages: 977 msg += f'\n - {p}' 978 if not silent: 979 print(msg) 980 981 if _uninstall and not _from_completely_uninstall and not use_uv_pip: 982 for install_name in _packages: 983 _install_no_version = get_install_no_version(install_name) 984 if _install_no_version in ('pip', 'wheel', 'uv'): 985 continue 986 if not completely_uninstall_package( 987 _install_no_version, 988 venv=venv, debug=debug, 989 ) and not silent: 990 warn( 991 f"Failed to clean up package '{_install_no_version}'.", 992 ) 993 994 ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease. 995 if use_uv_pip: 996 _args.insert(0, 'pip') 997 if not _uninstall and get_prerelease_dependencies(_packages): 998 _args.append('--prerelease=allow') 999 1000 rc = run_python_package( 1001 ('pip' if not use_uv_pip else 'uv'), 1002 _args + _packages, 1003 venv=None, 1004 env=_get_pip_os_env(color=color), 1005 debug=debug, 1006 ) 1007 if debug: 1008 print(f"{rc=}") 1009 success = rc == 0 1010 1011 msg = ( 1012 "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 1013 else "Failed to " + ('un' if _uninstall else '') + "install packages." 1014 ) 1015 if not silent: 1016 print(msg) 1017 if debug and not silent: 1018 print('pip ' + ('un' if _uninstall else '') + 'install returned:', success) 1019 return success 1020 1021 1022def get_prerelease_dependencies(_packages: Optional[List[str]] = None): 1023 """ 1024 Return a list of explicitly prerelease dependencies from a list of packages. 1025 """ 1026 if _packages is None: 1027 _packages = list(all_packages.keys()) 1028 prelrease_strings = ['dev', 'rc', 'a'] 1029 prerelease_packages = [] 1030 for install_name in _packages: 1031 _install_no_version = get_install_no_version(install_name) 1032 import_name = _install_to_import_name(install_name) 1033 install_with_version = _import_to_install_name(import_name) 1034 version_only = ( 1035 install_with_version.lower().replace(_install_no_version.lower(), '') 1036 .split(']')[-1] 1037 ) 1038 1039 is_prerelease = False 1040 for prelrease_string in prelrease_strings: 1041 if prelrease_string in version_only: 1042 is_prerelease = True 1043 1044 if is_prerelease: 1045 prerelease_packages.append(install_name) 1046 return prerelease_packages 1047 1048 1049def completely_uninstall_package( 1050 install_name: str, 1051 venv: str = 'mrsm', 1052 debug: bool = False, 1053 ) -> bool: 1054 """ 1055 Continue calling `pip uninstall` until a package is completely 1056 removed from a virtual environment. 1057 This is useful for dealing with multiple installed versions of a package. 1058 """ 1059 attempts = 0 1060 _install_no_version = get_install_no_version(install_name) 1061 clean_install_no_version = _install_no_version.lower().replace('-', '_') 1062 installed_versions = [] 1063 vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug) 1064 if not vtp.exists(): 1065 return True 1066 1067 for file_name in os.listdir(vtp): 1068 if not file_name.endswith('.dist-info'): 1069 continue 1070 clean_dist_info = file_name.replace('-', '_').lower() 1071 if not clean_dist_info.startswith(clean_install_no_version): 1072 continue 1073 installed_versions.append(file_name) 1074 1075 max_attempts = len(installed_versions) 1076 while attempts < max_attempts: 1077 if not venv_contains_package( 1078 _install_to_import_name(_install_no_version), 1079 venv=venv, debug=debug, 1080 ): 1081 return True 1082 if not pip_uninstall( 1083 _install_no_version, 1084 venv = venv, 1085 silent = (not debug), 1086 _from_completely_uninstall = True, 1087 debug = debug, 1088 ): 1089 return False 1090 attempts += 1 1091 return False 1092 1093 1094def pip_uninstall( 1095 *args, **kw 1096 ) -> bool: 1097 """ 1098 Uninstall Python packages. 1099 This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`. 1100 """ 1101 return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'}) 1102 1103 1104def run_python_package( 1105 package_name: str, 1106 args: Optional[List[str]] = None, 1107 venv: Optional[str] = 'mrsm', 1108 cwd: Optional[str] = None, 1109 foreground: bool = False, 1110 as_proc: bool = False, 1111 capture_output: bool = False, 1112 debug: bool = False, 1113 **kw: Any, 1114 ) -> Union[int, subprocess.Popen, None]: 1115 """ 1116 Runs an installed python package. 1117 E.g. Translates to `/usr/bin/python -m [package]` 1118 1119 Parameters 1120 ---------- 1121 package_name: str 1122 The Python module to be executed. 1123 1124 args: Optional[List[str]], default None 1125 Additional command line arguments to be appended after `-m [package]`. 1126 1127 venv: Optional[str], default 'mrsm' 1128 If specified, execute the Python interpreter from a virtual environment. 1129 1130 cwd: Optional[str], default None 1131 If specified, change directories before starting the process. 1132 Defaults to `None`. 1133 1134 as_proc: bool, default False 1135 If `True`, return a `subprocess.Popen` object. 1136 1137 capture_output: bool, default False 1138 If `as_proc` is `True`, capture stdout and stderr. 1139 1140 foreground: bool, default False 1141 If `True`, start the subprocess as a foreground process. 1142 Defaults to `False`. 1143 1144 kw: Any 1145 Additional keyword arguments to pass to `meerschaum.utils.process.run_process()` 1146 and by extension `subprocess.Popen()`. 1147 1148 Returns 1149 ------- 1150 Either a return code integer or a `subprocess.Popen` object 1151 (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`). 1152 """ 1153 import sys, platform 1154 import subprocess 1155 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 1156 from meerschaum.utils.process import run_process 1157 from meerschaum.utils.warnings import warn 1158 if args is None: 1159 args = [] 1160 old_cwd = os.getcwd() 1161 if cwd is not None: 1162 os.chdir(cwd) 1163 executable = venv_executable(venv=venv) 1164 venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None 1165 env_dict = kw.get('env', os.environ).copy() 1166 if venv_path is not None: 1167 env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()}) 1168 command = [executable, '-m', str(package_name)] + [str(a) for a in args] 1169 import traceback 1170 if debug: 1171 print(command, file=sys.stderr) 1172 try: 1173 to_return = run_process( 1174 command, 1175 foreground = foreground, 1176 as_proc = as_proc, 1177 capture_output = capture_output, 1178 **kw 1179 ) 1180 except Exception as e: 1181 msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}" 1182 warn(msg, color=False) 1183 stdout, stderr = ( 1184 (None, None) 1185 if not capture_output 1186 else (subprocess.PIPE, subprocess.PIPE) 1187 ) 1188 proc = subprocess.Popen( 1189 command, 1190 stdout = stdout, 1191 stderr = stderr, 1192 env = env_dict, 1193 ) 1194 to_return = proc if as_proc else proc.wait() 1195 except KeyboardInterrupt: 1196 to_return = 1 if not as_proc else None 1197 os.chdir(old_cwd) 1198 return to_return 1199 1200 1201def attempt_import( 1202 *names: str, 1203 lazy: bool = True, 1204 warn: bool = True, 1205 install: bool = True, 1206 venv: Optional[str] = 'mrsm', 1207 precheck: bool = True, 1208 split: bool = True, 1209 check_update: bool = False, 1210 check_pypi: bool = False, 1211 check_is_installed: bool = True, 1212 allow_outside_venv: bool = True, 1213 color: bool = True, 1214 debug: bool = False 1215 ) -> Any: 1216 """ 1217 Raise a warning if packages are not installed; otherwise import and return modules. 1218 If `lazy` is `True`, return lazy-imported modules. 1219 1220 Returns tuple of modules if multiple names are provided, else returns one module. 1221 1222 Parameters 1223 ---------- 1224 names: List[str] 1225 The packages to be imported. 1226 1227 lazy: bool, default True 1228 If `True`, lazily load packages. 1229 1230 warn: bool, default True 1231 If `True`, raise a warning if a package cannot be imported. 1232 1233 install: bool, default True 1234 If `True`, attempt to install a missing package into the designated virtual environment. 1235 If `check_update` is True, install updates if available. 1236 1237 venv: Optional[str], default 'mrsm' 1238 The virtual environment in which to search for packages and to install packages into. 1239 1240 precheck: bool, default True 1241 If `True`, attempt to find module before importing (necessary for checking if modules exist 1242 and retaining lazy imports), otherwise assume lazy is `False`. 1243 1244 split: bool, default True 1245 If `True`, split packages' names on `'.'`. 1246 1247 check_update: bool, default False 1248 If `True` and `install` is `True`, install updates if the required minimum version 1249 does not match. 1250 1251 check_pypi: bool, default False 1252 If `True` and `check_update` is `True`, check PyPI when determining whether 1253 an update is required. 1254 1255 check_is_installed: bool, default True 1256 If `True`, check if the package is contained in the virtual environment. 1257 1258 allow_outside_venv: bool, default True 1259 If `True`, search outside of the specified virtual environment 1260 if the package cannot be found. 1261 Setting to `False` will reinstall the package into a virtual environment, even if it 1262 is installed outside. 1263 1264 color: bool, default True 1265 If `False`, do not print ANSI colors. 1266 1267 Returns 1268 ------- 1269 The specified modules. If they're not available and `install` is `True`, it will first 1270 download them into a virtual environment and return the modules. 1271 1272 Examples 1273 -------- 1274 >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy') 1275 >>> pandas = attempt_import('pandas') 1276 1277 """ 1278 1279 import importlib.util 1280 1281 ### to prevent recursion, check if parent Meerschaum package is being imported 1282 if names == ('meerschaum',): 1283 return _import_module('meerschaum') 1284 1285 if venv == 'mrsm' and _import_hook_venv is not None: 1286 if debug: 1287 print(f"Import hook for virtual environment '{_import_hook_venv}' is active.") 1288 venv = _import_hook_venv 1289 1290 _warnings = _import_module('meerschaum.utils.warnings') 1291 warn_function = _warnings.warn 1292 1293 def do_import(_name: str, **kw) -> Union['ModuleType', None]: 1294 with Venv(venv=venv, debug=debug): 1295 ### determine the import method (lazy vs normal) 1296 from meerschaum.utils.misc import filter_keywords 1297 import_method = ( 1298 _import_module if not lazy 1299 else lazy_import 1300 ) 1301 try: 1302 mod = import_method(_name, **(filter_keywords(import_method, **kw))) 1303 except Exception as e: 1304 if warn: 1305 import traceback 1306 traceback.print_exception(type(e), e, e.__traceback__) 1307 warn_function( 1308 f"Failed to import module '{_name}'.\nException:\n{e}", 1309 ImportWarning, 1310 stacklevel = (5 if lazy else 4), 1311 color = False, 1312 ) 1313 mod = None 1314 return mod 1315 1316 modules = [] 1317 for name in names: 1318 ### Check if package is a declared dependency. 1319 root_name = name.split('.')[0] if split else name 1320 install_name = _import_to_install_name(root_name) 1321 1322 if install_name is None: 1323 install_name = root_name 1324 if warn and root_name != 'plugins': 1325 warn_function( 1326 f"Package '{root_name}' is not declared in meerschaum.utils.packages.", 1327 ImportWarning, 1328 stacklevel = 3, 1329 color = False 1330 ) 1331 1332 ### Determine if the package exists. 1333 if precheck is False: 1334 found_module = ( 1335 do_import( 1336 name, debug=debug, warn=False, venv=venv, color=color, 1337 check_update=False, check_pypi=False, split=split, 1338 ) is not None 1339 ) 1340 else: 1341 if check_is_installed: 1342 with _locks['_is_installed_first_check']: 1343 if not _is_installed_first_check.get(name, False): 1344 package_is_installed = is_installed( 1345 name, 1346 venv = venv, 1347 split = split, 1348 allow_outside_venv = allow_outside_venv, 1349 debug = debug, 1350 ) 1351 _is_installed_first_check[name] = package_is_installed 1352 else: 1353 package_is_installed = _is_installed_first_check[name] 1354 else: 1355 package_is_installed = _is_installed_first_check.get( 1356 name, 1357 venv_contains_package(name, venv=venv, split=split, debug=debug) 1358 ) 1359 found_module = package_is_installed 1360 1361 if not found_module: 1362 if install: 1363 if not pip_install( 1364 install_name, 1365 venv = venv, 1366 split = False, 1367 check_update = check_update, 1368 color = color, 1369 debug = debug 1370 ) and warn: 1371 warn_function( 1372 f"Failed to install '{install_name}'.", 1373 ImportWarning, 1374 stacklevel = 3, 1375 color = False, 1376 ) 1377 elif warn: 1378 ### Raise a warning if we can't find the package and install = False. 1379 warn_function( 1380 (f"\n\nMissing package '{name}' from virtual environment '{venv}'; " 1381 + "some features will not work correctly." 1382 + f"\n\nSet install=True when calling attempt_import.\n"), 1383 ImportWarning, 1384 stacklevel = 3, 1385 color = False, 1386 ) 1387 1388 ### Do the import. Will be lazy if lazy=True. 1389 m = do_import( 1390 name, debug=debug, warn=warn, venv=venv, color=color, 1391 check_update=check_update, check_pypi=check_pypi, install=install, split=split, 1392 ) 1393 modules.append(m) 1394 1395 modules = tuple(modules) 1396 if len(modules) == 1: 1397 return modules[0] 1398 return modules 1399 1400 1401def lazy_import( 1402 name: str, 1403 local_name: str = None, 1404 **kw 1405 ) -> meerschaum.utils.packages.lazy_loader.LazyLoader: 1406 """ 1407 Lazily import a package. 1408 """ 1409 from meerschaum.utils.packages.lazy_loader import LazyLoader 1410 if local_name is None: 1411 local_name = name 1412 return LazyLoader( 1413 local_name, 1414 globals(), 1415 name, 1416 **kw 1417 ) 1418 1419 1420def pandas_name() -> str: 1421 """ 1422 Return the configured name for `pandas`. 1423 1424 Below are the expected possible values: 1425 1426 - 'pandas' 1427 - 'modin.pandas' 1428 - 'dask.dataframe' 1429 1430 """ 1431 from meerschaum.config import get_config 1432 pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True) 1433 if pandas_module_name == 'modin': 1434 pandas_module_name = 'modin.pandas' 1435 elif pandas_module_name == 'dask': 1436 pandas_module_name = 'dask.dataframe' 1437 1438 return pandas_module_name 1439 1440 1441emitted_pandas_warning: bool = False 1442def import_pandas( 1443 debug: bool = False, 1444 lazy: bool = False, 1445 **kw 1446 ) -> 'ModuleType': 1447 """ 1448 Quality-of-life function to attempt to import the configured version of `pandas`. 1449 """ 1450 import sys 1451 pandas_module_name = pandas_name() 1452 global emitted_pandas_warning 1453 1454 if pandas_module_name != 'pandas': 1455 with _locks['emitted_pandas_warning']: 1456 if not emitted_pandas_warning: 1457 from meerschaum.utils.warnings import warn 1458 emitted_pandas_warning = True 1459 warn( 1460 ( 1461 "You are using an alternative Pandas implementation " 1462 + f"'{pandas_module_name}'" 1463 + "\n Features may not work as expected." 1464 ), 1465 stack = False, 1466 ) 1467 1468 pytz = attempt_import('pytz', debug=debug, lazy=False, **kw) 1469 pandas = attempt_import('pandas', debug=debug, lazy=False, **kw) 1470 pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw) 1471 return pd 1472 1473 1474def import_rich( 1475 lazy: bool = True, 1476 debug: bool = False, 1477 **kw : Any 1478 ) -> 'ModuleType': 1479 """ 1480 Quality of life function for importing `rich`. 1481 """ 1482 from meerschaum.utils.formatting import ANSI, UNICODE 1483 if not ANSI and not UNICODE: 1484 return None 1485 1486 ## need typing_extensions for `from rich import box` 1487 typing_extensions = attempt_import( 1488 'typing_extensions', lazy=False, debug=debug 1489 ) 1490 pygments = attempt_import( 1491 'pygments', lazy=False, 1492 ) 1493 rich = attempt_import( 1494 'rich', lazy=lazy, 1495 **kw 1496 ) 1497 return rich 1498 1499 1500def _dash_less_than_2(**kw) -> bool: 1501 dash = attempt_import('dash', **kw) 1502 if dash is None: 1503 return None 1504 packaging_version = attempt_import('packaging.version', **kw) 1505 return ( 1506 packaging_version.parse(dash.__version__) < 1507 packaging_version.parse('2.0.0') 1508 ) 1509 1510 1511def import_dcc(warn=False, **kw) -> 'ModuleType': 1512 """ 1513 Import Dash Core Components (`dcc`). 1514 """ 1515 return ( 1516 attempt_import('dash_core_components', warn=warn, **kw) 1517 if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw) 1518 ) 1519 1520 1521def import_html(warn=False, **kw) -> 'ModuleType': 1522 """ 1523 Import Dash HTML Components (`html`). 1524 """ 1525 return ( 1526 attempt_import('dash_html_components', warn=warn, **kw) 1527 if _dash_less_than_2(warn=warn, **kw) 1528 else attempt_import('dash.html', warn=warn, **kw) 1529 ) 1530 1531 1532def get_modules_from_package( 1533 package: 'package', 1534 names: bool = False, 1535 recursive: bool = False, 1536 lazy: bool = False, 1537 modules_venvs: bool = False, 1538 debug: bool = False 1539 ): 1540 """ 1541 Find and import all modules in a package. 1542 1543 Returns 1544 ------- 1545 Either list of modules or tuple of lists. 1546 """ 1547 from os.path import dirname, join, isfile, isdir, basename 1548 import glob 1549 1550 pattern = '*' if recursive else '*.py' 1551 package_path = dirname(package.__file__ or package.__path__[0]) 1552 module_names = glob.glob(join(package_path, pattern), recursive=recursive) 1553 _all = [ 1554 basename(f)[:-3] if isfile(f) else basename(f) 1555 for f in module_names 1556 if ((isfile(f) and f.endswith('.py')) or isdir(f)) 1557 and not f.endswith('__init__.py') 1558 and not f.endswith('__pycache__') 1559 ] 1560 1561 if debug: 1562 from meerschaum.utils.debug import dprint 1563 dprint(str(_all)) 1564 modules = [] 1565 for module_name in [package.__name__ + "." + mod_name for mod_name in _all]: 1566 ### there's probably a better way than a try: catch but it'll do for now 1567 try: 1568 ### if specified, activate the module's virtual environment before importing. 1569 ### NOTE: this only considers the filename, so two modules from different packages 1570 ### may end up sharing virtual environments. 1571 if modules_venvs: 1572 activate_venv(module_name.split('.')[-1], debug=debug) 1573 m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name) 1574 modules.append(m) 1575 except Exception as e: 1576 if debug: 1577 dprint(str(e)) 1578 finally: 1579 if modules_venvs: 1580 deactivate_venv(module_name.split('.')[-1], debug=debug) 1581 if names: 1582 return _all, modules 1583 1584 return modules 1585 1586 1587def import_children( 1588 package: Optional['ModuleType'] = None, 1589 package_name: Optional[str] = None, 1590 types : Optional[List[str]] = None, 1591 lazy: bool = True, 1592 recursive: bool = False, 1593 debug: bool = False 1594 ) -> List['ModuleType']: 1595 """ 1596 Import all functions in a package to its `__init__`. 1597 1598 Parameters 1599 ---------- 1600 package: Optional[ModuleType], default None 1601 Package to import its functions into. 1602 If `None` (default), use parent. 1603 1604 package_name: Optional[str], default None 1605 Name of package to import its functions into 1606 If None (default), use parent. 1607 1608 types: Optional[List[str]], default None 1609 Types of members to return. 1610 Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']` 1611 1612 Returns 1613 ------- 1614 A list of modules. 1615 """ 1616 import sys, inspect 1617 1618 if types is None: 1619 types = ['method', 'builtin', 'function', 'class', 'module'] 1620 1621 ### if package_name and package are None, use parent 1622 if package is None and package_name is None: 1623 package_name = inspect.stack()[1][0].f_globals['__name__'] 1624 1625 ### populate package or package_name from other other 1626 if package is None: 1627 package = sys.modules[package_name] 1628 elif package_name is None: 1629 package_name = package.__name__ 1630 1631 ### Set attributes in sys module version of package. 1632 ### Kinda like setting a dictionary 1633 ### functions[name] = func 1634 modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug) 1635 _all, members = [], [] 1636 objects = [] 1637 for module in modules: 1638 _objects = [] 1639 for ob in inspect.getmembers(module): 1640 for t in types: 1641 ### ob is a tuple of (name, object) 1642 if getattr(inspect, 'is' + t)(ob[1]): 1643 _objects.append(ob) 1644 1645 if 'module' in types: 1646 _objects.append((module.__name__.split('.')[0], module)) 1647 objects += _objects 1648 for ob in objects: 1649 setattr(sys.modules[package_name], ob[0], ob[1]) 1650 _all.append(ob[0]) 1651 members.append(ob[1]) 1652 1653 if debug: 1654 from meerschaum.utils.debug import dprint 1655 dprint(str(_all)) 1656 ### set __all__ for import * 1657 setattr(sys.modules[package_name], '__all__', _all) 1658 return members 1659 1660 1661_reload_module_cache = {} 1662def reload_package( 1663 package: str, 1664 skip_submodules: Optional[List[str]] = None, 1665 lazy: bool = False, 1666 debug: bool = False, 1667 **kw: Any 1668 ): 1669 """ 1670 Recursively load a package's subpackages, even if they were not previously loaded. 1671 """ 1672 import sys 1673 if isinstance(package, str): 1674 package_name = package 1675 else: 1676 try: 1677 package_name = package.__name__ 1678 except Exception as e: 1679 package_name = str(package) 1680 1681 skip_submodules = skip_submodules or [] 1682 if 'meerschaum.utils.packages' not in skip_submodules: 1683 skip_submodules.append('meerschaum.utils.packages') 1684 def safeimport(): 1685 subs = [ 1686 m for m in sys.modules 1687 if m.startswith(package_name + '.') 1688 ] 1689 subs_to_skip = [] 1690 for skip_mod in skip_submodules: 1691 for mod in subs: 1692 if mod.startswith(skip_mod): 1693 subs_to_skip.append(mod) 1694 continue 1695 1696 subs = [m for m in subs if m not in subs_to_skip] 1697 for module_name in subs: 1698 _reload_module_cache[module_name] = sys.modules.pop(module_name, None) 1699 if not subs_to_skip: 1700 _reload_module_cache[package_name] = sys.modules.pop(package_name, None) 1701 1702 return _import_module(package_name) 1703 1704 return safeimport() 1705 1706 1707def reload_meerschaum(debug: bool = False) -> SuccessTuple: 1708 """ 1709 Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration. 1710 """ 1711 reload_package( 1712 'meerschaum', 1713 skip_submodules = [ 1714 'meerschaum._internal.shell', 1715 'meerschaum.utils.pool', 1716 ] 1717 ) 1718 1719 from meerschaum.plugins import reload_plugins 1720 from meerschaum._internal.shell.Shell import _insert_shell_actions 1721 reload_plugins(debug=debug) 1722 _insert_shell_actions() 1723 return True, "Success" 1724 1725 1726def is_installed( 1727 import_name: str, 1728 venv: Optional[str] = 'mrsm', 1729 split: bool = True, 1730 allow_outside_venv: bool = True, 1731 debug: bool = False, 1732 ) -> bool: 1733 """ 1734 Check whether a package is installed. 1735 1736 Parameters 1737 ---------- 1738 import_name: str 1739 The import name of the module. 1740 1741 venv: Optional[str], default 'mrsm' 1742 The venv in which to search for the module. 1743 1744 split: bool, default True 1745 If `True`, split on periods to determine the root module name. 1746 1747 allow_outside_venv: bool, default True 1748 If `True`, search outside of the specified virtual environment 1749 if the package cannot be found. 1750 """ 1751 if debug: 1752 from meerschaum.utils.debug import dprint 1753 root_name = import_name.split('.')[0] if split else import_name 1754 import importlib.util 1755 with Venv(venv, debug=debug): 1756 try: 1757 spec_path = pathlib.Path( 1758 get_module_path(root_name, venv=venv, debug=debug) 1759 or 1760 ( 1761 importlib.util.find_spec(root_name).origin 1762 if venv is not None and allow_outside_venv 1763 else None 1764 ) 1765 ) 1766 except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e: 1767 spec_path = None 1768 1769 found = ( 1770 not need_update( 1771 None, import_name = root_name, 1772 _run_determine_version = False, 1773 check_pypi = False, 1774 version = determine_version( 1775 spec_path, venv=venv, debug=debug, import_name=root_name 1776 ), 1777 debug = debug, 1778 ) 1779 ) if spec_path is not None else False 1780 1781 return found 1782 1783 1784def venv_contains_package( 1785 import_name: str, 1786 venv: Optional[str] = 'mrsm', 1787 split: bool = True, 1788 debug: bool = False, 1789 ) -> bool: 1790 """ 1791 Search the contents of a virtual environment for a package. 1792 """ 1793 import site 1794 import pathlib 1795 root_name = import_name.split('.')[0] if split else import_name 1796 return get_module_path(root_name, venv=venv, debug=debug) is not None 1797 1798 1799def package_venv(package: 'ModuleType') -> Union[str, None]: 1800 """ 1801 Inspect a package and return the virtual environment in which it presides. 1802 """ 1803 import os 1804 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 1805 if str(VIRTENV_RESOURCES_PATH) not in package.__file__: 1806 return None 1807 return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1] 1808 1809 1810def ensure_readline() -> 'ModuleType': 1811 """Make sure that the `readline` package is able to be imported.""" 1812 import sys 1813 try: 1814 import readline 1815 except ImportError: 1816 readline = None 1817 1818 if readline is None: 1819 import platform 1820 rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3" 1821 try: 1822 rl = attempt_import( 1823 rl_name, 1824 lazy = False, 1825 install = True, 1826 venv = None, 1827 warn = False, 1828 ) 1829 except (ImportError, ModuleNotFoundError): 1830 if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None): 1831 print(f"Unable to import {rl_name}!", file=sys.stderr) 1832 sys.exit(1) 1833 1834 sys.modules['readline'] = readline 1835 return readline 1836 1837_pkg_resources_get_distribution = None 1838_custom_distributions = {} 1839def _monkey_patch_get_distribution(_dist: str, _version: str) -> None: 1840 """ 1841 Monkey patch `pkg_resources.get_distribution` to allow for importing `flask_compress`. 1842 """ 1843 import pkg_resources 1844 from collections import namedtuple 1845 global _pkg_resources_get_distribution 1846 with _locks['_pkg_resources_get_distribution']: 1847 _pkg_resources_get_distribution = pkg_resources.get_distribution 1848 _custom_distributions[_dist] = _version 1849 _Dist = namedtuple('_Dist', ['version']) 1850 def _get_distribution(dist): 1851 """Hack for flask-compress.""" 1852 if dist in _custom_distributions: 1853 return _Dist(_custom_distributions[dist]) 1854 return _pkg_resources_get_distribution(dist) 1855 pkg_resources.get_distribution = _get_distribution 1856 1857 1858def _get_pip_os_env(color: bool = True): 1859 """ 1860 Return the environment variables context in which `pip` should be run. 1861 See PEP 668 for why we are overriding the environment. 1862 """ 1863 import os, sys, platform 1864 python_bin_path = pathlib.Path(sys.executable) 1865 pip_os_env = os.environ.copy() 1866 path_str = pip_os_env.get('PATH', '') or '' 1867 path_sep = ':' if platform.system() != 'Windows' else ';' 1868 pip_os_env.update({ 1869 'PIP_BREAK_SYSTEM_PACKAGES': 'true', 1870 ('FORCE_COLOR' if color else 'NO_COLOR'): '1', 1871 }) 1872 if str(python_bin_path) not in path_str: 1873 pip_os_env['PATH'] = str(python_bin_path.parent) + path_sep + path_str 1874 1875 return pip_os_env
41def get_module_path( 42 import_name: str, 43 venv: Optional[str] = 'mrsm', 44 debug: bool = False, 45 _try_install_name_on_fail: bool = True, 46 ) -> Union[pathlib.Path, None]: 47 """ 48 Get a module's path without importing. 49 """ 50 import site 51 if debug: 52 from meerschaum.utils.debug import dprint 53 if not _try_install_name_on_fail: 54 install_name = _import_to_install_name(import_name, with_version=False) 55 install_name_lower = install_name.lower().replace('-', '_') 56 import_name_lower = install_name_lower 57 else: 58 import_name_lower = import_name.lower().replace('-', '_') 59 60 vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug) 61 if not vtp.exists(): 62 if debug: 63 dprint( 64 ( 65 "Venv '{venv}' does not exist, cannot import " 66 + f"'{import_name}'." 67 ), 68 color = False, 69 ) 70 return None 71 72 venv_target_candidate_paths = [vtp] 73 if venv is None: 74 site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())] 75 site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()] 76 77 paths_to_add = [ 78 path 79 for path in site_user_packages_dirs + site_packages_dirs 80 if path not in venv_target_candidate_paths 81 ] 82 venv_target_candidate_paths += paths_to_add 83 84 candidates = [] 85 for venv_target_candidate in venv_target_candidate_paths: 86 try: 87 file_names = os.listdir(venv_target_candidate) 88 except FileNotFoundError: 89 continue 90 for file_name in file_names: 91 file_name_lower = file_name.lower().replace('-', '_') 92 if not file_name_lower.startswith(import_name_lower): 93 continue 94 if file_name.endswith('dist_info'): 95 continue 96 file_path = venv_target_candidate / file_name 97 98 ### Most likely: Is a directory with __init__.py 99 if file_name_lower == import_name_lower and file_path.is_dir(): 100 init_path = file_path / '__init__.py' 101 if init_path.exists(): 102 candidates.append(init_path) 103 104 ### May be a standalone .py file. 105 elif file_name_lower == import_name_lower + '.py': 106 candidates.append(file_path) 107 108 ### Compiled wheels (e.g. pyodbc) 109 elif file_name_lower.startswith(import_name_lower + '.'): 110 candidates.append(file_path) 111 112 if len(candidates) == 1: 113 return candidates[0] 114 115 if not candidates: 116 if _try_install_name_on_fail: 117 return get_module_path( 118 import_name, venv=venv, debug=debug, 119 _try_install_name_on_fail=False 120 ) 121 return None 122 123 specs_paths = [] 124 for candidate_path in candidates: 125 spec = importlib.util.spec_from_file_location(import_name, str(candidate_path)) 126 if spec is not None: 127 return candidate_path 128 129 return None
Get a module's path without importing.
132def manually_import_module( 133 import_name: str, 134 venv: Optional[str] = 'mrsm', 135 check_update: bool = True, 136 check_pypi: bool = False, 137 install: bool = True, 138 split: bool = True, 139 warn: bool = True, 140 color: bool = True, 141 debug: bool = False, 142 use_sys_modules: bool = True, 143 ) -> Union['ModuleType', None]: 144 """ 145 Manually import a module from a virtual environment (or the base environment). 146 147 Parameters 148 ---------- 149 import_name: str 150 The name of the module. 151 152 venv: Optional[str], default 'mrsm' 153 The virtual environment to read from. 154 155 check_update: bool, default True 156 If `True`, examine whether the available version of the package meets the required version. 157 158 check_pypi: bool, default False 159 If `True`, check PyPI for updates before importing. 160 161 install: bool, default True 162 If `True`, install the package if it's not installed or needs an update. 163 164 split: bool, default True 165 If `True`, split `import_name` on periods to get the package name. 166 167 warn: bool, default True 168 If `True`, raise a warning if the package cannot be imported. 169 170 color: bool, default True 171 If `True`, use color output for debug and warning text. 172 173 debug: bool, default False 174 Verbosity toggle. 175 176 use_sys_modules: bool, default True 177 If `True`, return the module in `sys.modules` if it exists. 178 Otherwise continue with manually importing. 179 180 Returns 181 ------- 182 The specified module or `None` if it can't be imported. 183 184 """ 185 import sys 186 _previously_imported = import_name in sys.modules 187 if _previously_imported and use_sys_modules: 188 return sys.modules[import_name] 189 if debug: 190 from meerschaum.utils.debug import dprint 191 from meerschaum.utils.warnings import warn as warn_function 192 import warnings 193 root_name = import_name.split('.')[0] if split else import_name 194 install_name = _import_to_install_name(root_name) 195 196 root_path = get_module_path(root_name, venv=venv) 197 if root_path is None: 198 return None 199 mod_path = root_path 200 if mod_path.is_dir(): 201 for _dir in import_name.split('.')[:-1]: 202 mod_path = mod_path / _dir 203 possible_end_module_filename = import_name.split('.')[-1] + '.py' 204 try: 205 mod_path = ( 206 (mod_path / possible_end_module_filename) 207 if possible_end_module_filename in os.listdir(mod_path) 208 else ( 209 mod_path / import_name.split('.')[-1] / '__init__.py' 210 ) 211 ) 212 except Exception as e: 213 mod_path = None 214 215 spec = ( 216 importlib.util.find_spec(import_name) if mod_path is None or not mod_path.exists() 217 else importlib.util.spec_from_file_location(import_name, str(mod_path)) 218 ) 219 root_spec = ( 220 importlib.util.find_spec(root_name) if not root_path.exists() 221 else importlib.util.spec_from_file_location(root_name, str(root_path)) 222 ) 223 224 ### Check for updates before importing. 225 _version = ( 226 determine_version( 227 pathlib.Path(root_spec.origin), 228 import_name=root_name, venv=venv, debug=debug 229 ) if root_spec is not None and root_spec.origin is not None else None 230 ) 231 232 if _version is not None: 233 if check_update: 234 if need_update( 235 None, 236 import_name = root_name, 237 version = _version, 238 check_pypi = check_pypi, 239 debug = debug, 240 ): 241 if install: 242 if not pip_install( 243 root_name, 244 venv = venv, 245 split = False, 246 check_update = check_update, 247 color = color, 248 debug = debug 249 ) and warn: 250 warn_function( 251 f"There's an update available for '{install_name}', " 252 + "but it failed to install. " 253 + "Try installig via Meerschaum with " 254 + "`install packages '{install_name}'`.", 255 ImportWarning, 256 stacklevel = 3, 257 color = False, 258 ) 259 elif warn: 260 warn_function( 261 f"There's an update available for '{root_name}'.", 262 stack = False, 263 color = False, 264 ) 265 spec = ( 266 importlib.util.find_spec(import_name) 267 if mod_path is None or not mod_path.exists() 268 else importlib.util.spec_from_file_location(import_name, str(mod_path)) 269 ) 270 271 272 if spec is None: 273 try: 274 mod = _import_module(import_name) 275 except Exception as e: 276 mod = None 277 return mod 278 279 with Venv(venv, debug=debug): 280 mod = importlib.util.module_from_spec(spec) 281 old_sys_mod = sys.modules.get(import_name, None) 282 sys.modules[import_name] = mod 283 284 try: 285 with warnings.catch_warnings(): 286 warnings.filterwarnings('ignore', 'The NumPy') 287 spec.loader.exec_module(mod) 288 except Exception as e: 289 pass 290 mod = _import_module(import_name) 291 if old_sys_mod is not None: 292 sys.modules[import_name] = old_sys_mod 293 else: 294 del sys.modules[import_name] 295 296 return mod
Manually import a module from a virtual environment (or the base environment).
Parameters
- import_name (str): The name of the module.
- venv (Optional[str], default 'mrsm'): The virtual environment to read from.
- check_update (bool, default True):
If
True
, examine whether the available version of the package meets the required version. - check_pypi (bool, default False):
If
True
, check PyPI for updates before importing. - install (bool, default True):
If
True
, install the package if it's not installed or needs an update. - split (bool, default True):
If
True
, splitimport_name
on periods to get the package name. - warn (bool, default True):
If
True
, raise a warning if the package cannot be imported. - color (bool, default True):
If
True
, use color output for debug and warning text. - debug (bool, default False): Verbosity toggle.
- use_sys_modules (bool, default True):
If
True
, return the module insys.modules
if it exists. Otherwise continue with manually importing.
Returns
- The specified module or
None
if it can't be imported.
327def get_install_no_version(install_name: str) -> str: 328 """ 329 Strip the version information from the install name. 330 """ 331 import re 332 return re.split(r'[\[=<>,! \]]', install_name)[0]
Strip the version information from the install name.
336def determine_version( 337 path: pathlib.Path, 338 import_name: Optional[str] = None, 339 venv: Optional[str] = 'mrsm', 340 search_for_metadata: bool = True, 341 split: bool = True, 342 warn: bool = False, 343 debug: bool = False, 344 ) -> Union[str, None]: 345 """ 346 Determine a module's `__version__` string from its filepath. 347 348 First it searches for pip metadata, then it attempts to import the module in a subprocess. 349 350 Parameters 351 ---------- 352 path: pathlib.Path 353 The file path of the module. 354 355 import_name: Optional[str], default None 356 The name of the module. If omitted, it will be determined from the file path. 357 Defaults to `None`. 358 359 venv: Optional[str], default 'mrsm' 360 The virtual environment of the Python interpreter to use if importing is necessary. 361 362 search_for_metadata: bool, default True 363 If `True`, search the pip site_packages directory (assumed to be the parent) 364 for the corresponding dist-info directory. 365 366 warn: bool, default True 367 If `True`, raise a warning if the module fails to import in the subprocess. 368 369 split: bool, default True 370 If `True`, split the determined import name by periods to get the room name. 371 372 Returns 373 ------- 374 The package's version string if available or `None`. 375 If multiple versions are found, it will trigger an import in a subprocess. 376 377 """ 378 with _locks['import_versions']: 379 if venv not in import_versions: 380 import_versions[venv] = {} 381 import importlib.metadata 382 import re, os 383 old_cwd = os.getcwd() 384 if debug: 385 from meerschaum.utils.debug import dprint 386 from meerschaum.utils.warnings import warn as warn_function 387 if import_name is None: 388 import_name = path.parent.stem if path.stem == '__init__' else path.stem 389 import_name = import_name.split('.')[0] if split else import_name 390 if import_name in import_versions[venv]: 391 return import_versions[venv][import_name] 392 _version = None 393 module_parent_dir = ( 394 path.parent.parent if path.stem == '__init__' else path.parent 395 ) if path is not None else venv_target_path(venv, debug=debug) 396 397 installed_dir_name = _import_to_dir_name(import_name) 398 clean_installed_dir_name = installed_dir_name.lower().replace('-', '_') 399 400 ### First, check if a dist-info directory exists. 401 _found_versions = [] 402 if search_for_metadata: 403 for filename in os.listdir(module_parent_dir): 404 if not filename.endswith('.dist-info'): 405 continue 406 filename_lower = filename.lower() 407 if not filename_lower.startswith(clean_installed_dir_name + '-'): 408 continue 409 _v = filename.replace('.dist-info', '').split("-")[-1] 410 _found_versions.append(_v) 411 412 if len(_found_versions) == 1: 413 _version = _found_versions[0] 414 with _locks['import_versions']: 415 import_versions[venv][import_name] = _version 416 return _found_versions[0] 417 418 if not _found_versions: 419 try: 420 import importlib.metadata as importlib_metadata 421 except ImportError: 422 importlib_metadata = attempt_import( 423 'importlib_metadata', 424 debug=debug, check_update=False, precheck=False, 425 color=False, check_is_installed=False, lazy=False, 426 ) 427 try: 428 os.chdir(module_parent_dir) 429 _version = importlib_metadata.metadata(import_name)['Version'] 430 except Exception as e: 431 _version = None 432 finally: 433 os.chdir(old_cwd) 434 435 if _version is not None: 436 with _locks['import_versions']: 437 import_versions[venv][import_name] = _version 438 return _version 439 440 if debug: 441 print(f'Found multiple versions for {import_name}: {_found_versions}') 442 443 module_parent_dir_str = module_parent_dir.as_posix() 444 445 ### Not a pip package, so let's try importing the module directly (in a subprocess). 446 _no_version_str = 'no-version' 447 code = ( 448 f"import sys, importlib; sys.path.insert(0, '{module_parent_dir_str}');\n" 449 + f"module = importlib.import_module('{import_name}');\n" 450 + "try:\n" 451 + " print(module.__version__ , end='')\n" 452 + "except:\n" 453 + f" print('{_no_version_str}', end='')" 454 ) 455 exit_code, stdout_bytes, stderr_bytes = venv_exec( 456 code, venv=venv, with_extras=True, debug=debug 457 ) 458 stdout, stderr = stdout_bytes.decode('utf-8'), stderr_bytes.decode('utf-8') 459 _version = stdout.split('\n')[-1] if exit_code == 0 else None 460 _version = _version if _version != _no_version_str else None 461 462 if _version is None: 463 _version = _get_package_metadata(import_name, venv).get('version', None) 464 if _version is None and warn: 465 warn_function( 466 f"Failed to determine a version for '{import_name}':\n{stderr}", 467 stack = False 468 ) 469 470 ### If `__version__` doesn't exist, return `None`. 471 import_versions[venv][import_name] = _version 472 return _version
Determine a module's __version__
string from its filepath.
First it searches for pip metadata, then it attempts to import the module in a subprocess.
Parameters
- path (pathlib.Path): The file path of the module.
- import_name (Optional[str], default None):
The name of the module. If omitted, it will be determined from the file path.
Defaults to
None
. - venv (Optional[str], default 'mrsm'): The virtual environment of the Python interpreter to use if importing is necessary.
- search_for_metadata (bool, default True):
If
True
, search the pip site_packages directory (assumed to be the parent) for the corresponding dist-info directory. - warn (bool, default True):
If
True
, raise a warning if the module fails to import in the subprocess. - split (bool, default True):
If
True
, split the determined import name by periods to get the room name.
Returns
- The package's version string if available or
None
. - If multiple versions are found, it will trigger an import in a subprocess.
518def need_update( 519 package: Optional['ModuleType'] = None, 520 install_name: Optional[str] = None, 521 import_name: Optional[str] = None, 522 version: Optional[str] = None, 523 check_pypi: bool = False, 524 split: bool = True, 525 color: bool = True, 526 debug: bool = False, 527 _run_determine_version: bool = True, 528 ) -> bool: 529 """ 530 Check if a Meerschaum dependency needs an update. 531 Returns a bool for whether or not a package needs to be updated. 532 533 Parameters 534 ---------- 535 package: 'ModuleType' 536 The module of the package to be updated. 537 538 install_name: Optional[str], default None 539 If provided, use this string to determine the required version. 540 Otherwise use the install name defined in `meerschaum.utils.packages._packages`. 541 542 import_name: 543 If provided, override the package's `__name__` string. 544 545 version: Optional[str], default None 546 If specified, override the package's `__version__` string. 547 548 check_pypi: bool, default False 549 If `True`, check pypi.org for updates. 550 Defaults to `False`. 551 552 split: bool, default True 553 If `True`, split the module's name on periods to detrive the root name. 554 Defaults to `True`. 555 556 color: bool, default True 557 If `True`, format debug output. 558 Defaults to `True`. 559 560 debug: bool, default True 561 Verbosity toggle. 562 563 Returns 564 ------- 565 A bool indicating whether the package requires an update. 566 567 """ 568 if debug: 569 from meerschaum.utils.debug import dprint 570 from meerschaum.utils.warnings import warn as warn_function 571 import re 572 root_name = ( 573 package.__name__.split('.')[0] if split else package.__name__ 574 ) if import_name is None else ( 575 import_name.split('.')[0] if split else import_name 576 ) 577 install_name = install_name or _import_to_install_name(root_name) 578 with _locks['_checked_for_updates']: 579 if install_name in _checked_for_updates: 580 return False 581 _checked_for_updates.add(install_name) 582 583 _install_no_version = get_install_no_version(install_name) 584 required_version = install_name.replace(_install_no_version, '') 585 if ']' in required_version: 586 required_version = required_version.split(']')[1] 587 588 ### No minimum version was specified, and we're not going to check PyPI. 589 if not required_version and not check_pypi: 590 return False 591 592 ### NOTE: Sometimes (rarely), we depend on a development build of a package. 593 if '.dev' in required_version: 594 required_version = required_version.split('.dev')[0] 595 if version and '.dev' in version: 596 version = version.split('.dev')[0] 597 598 try: 599 if not version: 600 if not _run_determine_version: 601 version = determine_version( 602 pathlib.Path(package.__file__), 603 import_name=root_name, warn=False, debug=debug 604 ) 605 if version is None: 606 return False 607 except Exception as e: 608 if debug: 609 dprint(str(e), color=color) 610 dprint("No version could be determined from the installed package.", color=color) 611 return False 612 split_version = version.split('.') 613 last_part = split_version[-1] 614 if len(split_version) == 2: 615 version = '.'.join(split_version) + '.0' 616 elif 'dev' in last_part or 'rc' in last_part: 617 tag = 'dev' if 'dev' in last_part else 'rc' 618 last_sep = '-' 619 if not last_part.startswith(tag): 620 last_part = f'-{tag}'.join(last_part.split(tag)) 621 last_sep = '.' 622 version = '.'.join(split_version[:-1]) + last_sep + last_part 623 elif len(split_version) > 3: 624 version = '.'.join(split_version[:3]) 625 626 packaging_version = attempt_import( 627 'packaging.version', check_update=False, lazy=False, debug=debug, 628 ) 629 630 ### Get semver if necessary 631 if required_version: 632 semver_path = get_module_path('semver', debug=debug) 633 if semver_path is None: 634 pip_install(_import_to_install_name('semver'), debug=debug) 635 semver = attempt_import('semver', check_update=False, lazy=False, debug=debug) 636 if check_pypi: 637 ### Check PyPI for updates 638 update_checker = attempt_import( 639 'update_checker', lazy=False, check_update=False, debug=debug 640 ) 641 checker = update_checker.UpdateChecker() 642 result = checker.check(_install_no_version, version) 643 else: 644 ### Skip PyPI and assume we can't be sure. 645 result = None 646 647 ### Compare PyPI's version with our own. 648 if result is not None: 649 ### We have a result from PyPI and a stated required version. 650 if required_version: 651 try: 652 return semver.Version.parse(result.available_version).match(required_version) 653 except AttributeError as e: 654 pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug) 655 semver = manually_import_module('semver', venv='mrsm') 656 return semver.Version.parse(version).match(required_version) 657 except Exception as e: 658 if debug: 659 dprint(f"Failed to match versions with exception:\n{e}", color=color) 660 return False 661 662 ### If `check_pypi` and we don't have a required version, check if PyPI's version 663 ### is newer than the installed version. 664 else: 665 return ( 666 packaging_version.parse(result.available_version) > 667 packaging_version.parse(version) 668 ) 669 670 ### We might be depending on a prerelease. 671 ### Sanity check that the required version is not greater than the installed version. 672 required_version = ( 673 required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '') 674 .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '') 675 .replace('-py3-none-any.whl', '') 676 ) 677 678 if 'a' in required_version: 679 required_version = required_version.replace('a', '-dev').replace('+mrsm', '') 680 version = version.replace('a', '-dev').replace('+mrsm', '') 681 try: 682 return ( 683 (not semver.Version.parse(version).match(required_version)) 684 if required_version else False 685 ) 686 except AttributeError as e: 687 pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug) 688 semver = manually_import_module('semver', venv='mrsm', debug=debug) 689 return ( 690 (not semver.Version.parse(version).match(required_version)) 691 if required_version else False 692 ) 693 except Exception as e: 694 print(f"Unable to parse version ({version}) for package '{import_name}'.") 695 print(e) 696 if debug: 697 dprint(e) 698 return False 699 try: 700 return ( 701 packaging_version.parse(version) > 702 packaging_version.parse(required_version) 703 ) 704 except Exception as e: 705 if debug: 706 dprint(e) 707 return False 708 return False
Check if a Meerschaum dependency needs an update. Returns a bool for whether or not a package needs to be updated.
Parameters
- package ('ModuleType'): The module of the package to be updated.
- install_name (Optional[str], default None):
If provided, use this string to determine the required version.
Otherwise use the install name defined in
meerschaum.utils.packages._packages
. - import_name:: If provided, override the package's
__name__
string. - version (Optional[str], default None):
If specified, override the package's
__version__
string. - check_pypi (bool, default False):
If
True
, check pypi.org for updates. Defaults toFalse
. - split (bool, default True):
If
True
, split the module's name on periods to detrive the root name. Defaults toTrue
. - color (bool, default True):
If
True
, format debug output. Defaults toTrue
. - debug (bool, default True): Verbosity toggle.
Returns
- A bool indicating whether the package requires an update.
711def get_pip( 712 venv: Optional[str] = 'mrsm', 713 color: bool = True, 714 debug: bool = False, 715 ) -> bool: 716 """ 717 Download and run the get-pip.py script. 718 719 Parameters 720 ---------- 721 venv: Optional[str], default 'mrsm' 722 The virtual environment into which to install `pip`. 723 724 color: bool, default True 725 If `True`, force color output. 726 727 debug: bool, default False 728 Verbosity toggle. 729 730 Returns 731 ------- 732 A bool indicating success. 733 734 """ 735 import sys, subprocess 736 from meerschaum.utils.misc import wget 737 from meerschaum.config._paths import CACHE_RESOURCES_PATH 738 from meerschaum.config.static import STATIC_CONFIG 739 url = STATIC_CONFIG['system']['urls']['get-pip.py'] 740 dest = CACHE_RESOURCES_PATH / 'get-pip.py' 741 try: 742 wget(url, dest, color=False, debug=debug) 743 except Exception as e: 744 print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 745 sys.exit(1) 746 if venv is not None: 747 init_venv(venv=venv, debug=debug) 748 cmd_list = [venv_executable(venv=venv), dest.as_posix()] 749 return subprocess.call(cmd_list, env=_get_pip_os_env(color=color)) == 0
Download and run the get-pip.py script.
Parameters
- venv (Optional[str], default 'mrsm'):
The virtual environment into which to install
pip
. - color (bool, default True):
If
True
, force color output. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating success.
752def pip_install( 753 *install_names: str, 754 args: Optional[List[str]] = None, 755 requirements_file_path: Union[pathlib.Path, str, None] = None, 756 venv: Optional[str] = 'mrsm', 757 split: bool = False, 758 check_update: bool = True, 759 check_pypi: bool = True, 760 check_wheel: bool = True, 761 _uninstall: bool = False, 762 _from_completely_uninstall: bool = False, 763 _install_uv_pip: bool = True, 764 color: bool = True, 765 silent: bool = False, 766 debug: bool = False, 767 ) -> bool: 768 """ 769 Install packages from PyPI with `pip`. 770 771 Parameters 772 ---------- 773 *install_names: str 774 The installation names of packages to be installed. 775 This includes version restrictions. 776 Use `_import_to_install_name()` to get the predefined `install_name` for a package 777 from its import name. 778 779 args: Optional[List[str]], default None 780 A list of command line arguments to pass to `pip`. 781 If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`. 782 783 requirements_file_path: Optional[pathlib.Path, str], default None 784 If provided, append `['-r', '/path/to/requirements.txt']` to `args`. 785 786 venv: str, default 'mrsm' 787 The virtual environment to install into. 788 789 split: bool, default False 790 If `True`, split on periods and only install the root package name. 791 792 check_update: bool, default True 793 If `True`, check if the package requires an update. 794 795 check_pypi: bool, default True 796 If `True` and `check_update` is `True`, check PyPI for the latest version. 797 798 check_wheel: bool, default True 799 If `True`, check if `wheel` is available. 800 801 _uninstall: bool, default False 802 If `True`, uninstall packages instead. 803 804 color: bool, default True 805 If `True`, include color in debug text. 806 807 silent: bool, default False 808 If `True`, skip printing messages. 809 810 debug: bool, default False 811 Verbosity toggle. 812 813 Returns 814 ------- 815 A bool indicating success. 816 817 """ 818 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 819 from meerschaum.config import get_config 820 from meerschaum.config.static import STATIC_CONFIG 821 from meerschaum.utils.warnings import warn 822 from meerschaum.utils.misc import is_android 823 if args is None: 824 args = ['--upgrade'] if not _uninstall else [] 825 if color: 826 ANSI, UNICODE = True, True 827 else: 828 ANSI, UNICODE = False, False 829 if check_wheel: 830 have_wheel = venv_contains_package('wheel', venv=venv, debug=debug) 831 832 daemon_env_var = STATIC_CONFIG['environment']['daemon_id'] 833 inside_daemon = daemon_env_var in os.environ 834 if inside_daemon: 835 silent = True 836 837 _args = list(args) 838 have_pip = venv_contains_package('pip', venv=None, debug=debug) 839 try: 840 import pip 841 have_pip = True 842 except ImportError: 843 have_pip = False 844 try: 845 import uv 846 uv_bin = uv.find_uv_bin() 847 have_uv_pip = True 848 except (ImportError, FileNotFoundError): 849 uv_bin = None 850 have_uv_pip = False 851 if have_pip and not have_uv_pip and _install_uv_pip and not is_android(): 852 if not pip_install( 853 'uv', 854 venv=None, 855 debug=debug, 856 _install_uv_pip=False, 857 check_update=False, 858 check_pypi=False, 859 check_wheel=False, 860 ) and not silent: 861 warn( 862 f"Failed to install `uv` for virtual environment '{venv}'.", 863 color=False, 864 ) 865 866 use_uv_pip = ( 867 venv_contains_package('uv', venv=None, debug=debug) 868 and uv_bin is not None 869 and venv is not None 870 ) 871 872 import sys 873 if not have_pip and not use_uv_pip: 874 if not get_pip(venv=venv, color=color, debug=debug): 875 import sys 876 minor = sys.version_info.minor 877 print( 878 "\nFailed to import `pip` and `ensurepip`.\n" 879 + "If you are running Ubuntu/Debian, " 880 + "you might need to install `python3.{minor}-distutils`:\n\n" 881 + f" sudo apt install python3.{minor}-pip python3.{minor}-venv\n\n" 882 + "Please install pip and restart Meerschaum.\n\n" 883 + "You can find instructions on installing `pip` here:\n" 884 + "https://pip.pypa.io/en/stable/installing/" 885 ) 886 sys.exit(1) 887 888 with Venv(venv, debug=debug): 889 if venv is not None: 890 if ( 891 '--ignore-installed' not in args 892 and '-I' not in _args 893 and not _uninstall 894 and not use_uv_pip 895 ): 896 _args += ['--ignore-installed'] 897 if '--cache-dir' not in args and not _uninstall: 898 cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache' 899 _args += ['--cache-dir', str(cache_dir_path)] 900 901 if 'pip' not in ' '.join(_args) and not use_uv_pip: 902 if check_update and not _uninstall: 903 pip = attempt_import('pip', venv=venv, install=False, debug=debug, lazy=False) 904 if need_update(pip, check_pypi=check_pypi, debug=debug): 905 _args.append(all_packages['pip']) 906 907 _args = (['install'] if not _uninstall else ['uninstall']) + _args 908 909 if check_wheel and not _uninstall and not use_uv_pip: 910 if not have_wheel: 911 setup_packages_to_install = ( 912 ['setuptools', 'wheel'] 913 + ([] if is_android() else ['uv']) 914 ) 915 if not pip_install( 916 *setup_packages_to_install, 917 venv=venv, 918 check_update=False, 919 check_pypi=False, 920 check_wheel=False, 921 debug=debug, 922 _install_uv_pip=False, 923 ) and not silent: 924 from meerschaum.utils.misc import items_str 925 warn( 926 ( 927 f"Failed to install {items_str(setup_packages_to_install)} for virtual " 928 + f"environment '{venv}'." 929 ), 930 color=False, 931 ) 932 933 if requirements_file_path is not None: 934 _args.append('-r') 935 _args.append(pathlib.Path(requirements_file_path).resolve().as_posix()) 936 937 if not ANSI and '--no-color' not in _args: 938 _args.append('--no-color') 939 940 if '--no-input' not in _args and not use_uv_pip: 941 _args.append('--no-input') 942 943 if _uninstall and '-y' not in _args and not use_uv_pip: 944 _args.append('-y') 945 946 if '--no-warn-conflicts' not in _args and not _uninstall and not use_uv_pip: 947 _args.append('--no-warn-conflicts') 948 949 if '--disable-pip-version-check' not in _args and not use_uv_pip: 950 _args.append('--disable-pip-version-check') 951 952 if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall): 953 if venv is not None: 954 _args += ['--target', venv_target_path(venv, debug=debug)] 955 elif ( 956 '--target' not in _args 957 and '-t' not in _args 958 and not inside_venv() 959 and not _uninstall 960 and not use_uv_pip 961 ): 962 _args += ['--user'] 963 964 if debug: 965 if '-v' not in _args or '-vv' not in _args or '-vvv' not in _args: 966 if use_uv_pip: 967 _args.append('--verbose') 968 else: 969 if '-q' not in _args or '-qq' not in _args or '-qqq' not in _args: 970 pass 971 972 _packages = [ 973 (install_name if not _uninstall else get_install_no_version(install_name)) 974 for install_name in install_names 975 ] 976 msg = "Installing packages:" if not _uninstall else "Uninstalling packages:" 977 for p in _packages: 978 msg += f'\n - {p}' 979 if not silent: 980 print(msg) 981 982 if _uninstall and not _from_completely_uninstall and not use_uv_pip: 983 for install_name in _packages: 984 _install_no_version = get_install_no_version(install_name) 985 if _install_no_version in ('pip', 'wheel', 'uv'): 986 continue 987 if not completely_uninstall_package( 988 _install_no_version, 989 venv=venv, debug=debug, 990 ) and not silent: 991 warn( 992 f"Failed to clean up package '{_install_no_version}'.", 993 ) 994 995 ### NOTE: Only append the `--prerelease=allow` flag if we explicitly depend on a prerelease. 996 if use_uv_pip: 997 _args.insert(0, 'pip') 998 if not _uninstall and get_prerelease_dependencies(_packages): 999 _args.append('--prerelease=allow') 1000 1001 rc = run_python_package( 1002 ('pip' if not use_uv_pip else 'uv'), 1003 _args + _packages, 1004 venv=None, 1005 env=_get_pip_os_env(color=color), 1006 debug=debug, 1007 ) 1008 if debug: 1009 print(f"{rc=}") 1010 success = rc == 0 1011 1012 msg = ( 1013 "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 1014 else "Failed to " + ('un' if _uninstall else '') + "install packages." 1015 ) 1016 if not silent: 1017 print(msg) 1018 if debug and not silent: 1019 print('pip ' + ('un' if _uninstall else '') + 'install returned:', success) 1020 return success
Install packages from PyPI with pip
.
Parameters
- *install_names (str):
The installation names of packages to be installed.
This includes version restrictions.
Use
_import_to_install_name()
to get the predefinedinstall_name
for a package from its import name. - args (Optional[List[str]], default None):
A list of command line arguments to pass to
pip
. If not provided, default to['--upgrade']
if_uninstall
isFalse
, else[]
. - requirements_file_path (Optional[pathlib.Path, str], default None):
If provided, append
['-r', '/path/to/requirements.txt']
toargs
. - venv (str, default 'mrsm'): The virtual environment to install into.
- split (bool, default False):
If
True
, split on periods and only install the root package name. - check_update (bool, default True):
If
True
, check if the package requires an update. - check_pypi (bool, default True):
If
True
andcheck_update
isTrue
, check PyPI for the latest version. - check_wheel (bool, default True):
If
True
, check ifwheel
is available. - _uninstall (bool, default False):
If
True
, uninstall packages instead. - color (bool, default True):
If
True
, include color in debug text. - silent (bool, default False):
If
True
, skip printing messages. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating success.
1023def get_prerelease_dependencies(_packages: Optional[List[str]] = None): 1024 """ 1025 Return a list of explicitly prerelease dependencies from a list of packages. 1026 """ 1027 if _packages is None: 1028 _packages = list(all_packages.keys()) 1029 prelrease_strings = ['dev', 'rc', 'a'] 1030 prerelease_packages = [] 1031 for install_name in _packages: 1032 _install_no_version = get_install_no_version(install_name) 1033 import_name = _install_to_import_name(install_name) 1034 install_with_version = _import_to_install_name(import_name) 1035 version_only = ( 1036 install_with_version.lower().replace(_install_no_version.lower(), '') 1037 .split(']')[-1] 1038 ) 1039 1040 is_prerelease = False 1041 for prelrease_string in prelrease_strings: 1042 if prelrease_string in version_only: 1043 is_prerelease = True 1044 1045 if is_prerelease: 1046 prerelease_packages.append(install_name) 1047 return prerelease_packages
Return a list of explicitly prerelease dependencies from a list of packages.
1050def completely_uninstall_package( 1051 install_name: str, 1052 venv: str = 'mrsm', 1053 debug: bool = False, 1054 ) -> bool: 1055 """ 1056 Continue calling `pip uninstall` until a package is completely 1057 removed from a virtual environment. 1058 This is useful for dealing with multiple installed versions of a package. 1059 """ 1060 attempts = 0 1061 _install_no_version = get_install_no_version(install_name) 1062 clean_install_no_version = _install_no_version.lower().replace('-', '_') 1063 installed_versions = [] 1064 vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug) 1065 if not vtp.exists(): 1066 return True 1067 1068 for file_name in os.listdir(vtp): 1069 if not file_name.endswith('.dist-info'): 1070 continue 1071 clean_dist_info = file_name.replace('-', '_').lower() 1072 if not clean_dist_info.startswith(clean_install_no_version): 1073 continue 1074 installed_versions.append(file_name) 1075 1076 max_attempts = len(installed_versions) 1077 while attempts < max_attempts: 1078 if not venv_contains_package( 1079 _install_to_import_name(_install_no_version), 1080 venv=venv, debug=debug, 1081 ): 1082 return True 1083 if not pip_uninstall( 1084 _install_no_version, 1085 venv = venv, 1086 silent = (not debug), 1087 _from_completely_uninstall = True, 1088 debug = debug, 1089 ): 1090 return False 1091 attempts += 1 1092 return False
Continue calling pip uninstall
until a package is completely
removed from a virtual environment.
This is useful for dealing with multiple installed versions of a package.
1095def pip_uninstall( 1096 *args, **kw 1097 ) -> bool: 1098 """ 1099 Uninstall Python packages. 1100 This function is a wrapper around `pip_install()` but with `_uninstall` enforced as `True`. 1101 """ 1102 return pip_install(*args, _uninstall=True, **{k: v for k, v in kw.items() if k != '_uninstall'})
Uninstall Python packages.
This function is a wrapper around pip_install()
but with _uninstall
enforced as True
.
1105def run_python_package( 1106 package_name: str, 1107 args: Optional[List[str]] = None, 1108 venv: Optional[str] = 'mrsm', 1109 cwd: Optional[str] = None, 1110 foreground: bool = False, 1111 as_proc: bool = False, 1112 capture_output: bool = False, 1113 debug: bool = False, 1114 **kw: Any, 1115 ) -> Union[int, subprocess.Popen, None]: 1116 """ 1117 Runs an installed python package. 1118 E.g. Translates to `/usr/bin/python -m [package]` 1119 1120 Parameters 1121 ---------- 1122 package_name: str 1123 The Python module to be executed. 1124 1125 args: Optional[List[str]], default None 1126 Additional command line arguments to be appended after `-m [package]`. 1127 1128 venv: Optional[str], default 'mrsm' 1129 If specified, execute the Python interpreter from a virtual environment. 1130 1131 cwd: Optional[str], default None 1132 If specified, change directories before starting the process. 1133 Defaults to `None`. 1134 1135 as_proc: bool, default False 1136 If `True`, return a `subprocess.Popen` object. 1137 1138 capture_output: bool, default False 1139 If `as_proc` is `True`, capture stdout and stderr. 1140 1141 foreground: bool, default False 1142 If `True`, start the subprocess as a foreground process. 1143 Defaults to `False`. 1144 1145 kw: Any 1146 Additional keyword arguments to pass to `meerschaum.utils.process.run_process()` 1147 and by extension `subprocess.Popen()`. 1148 1149 Returns 1150 ------- 1151 Either a return code integer or a `subprocess.Popen` object 1152 (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`). 1153 """ 1154 import sys, platform 1155 import subprocess 1156 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 1157 from meerschaum.utils.process import run_process 1158 from meerschaum.utils.warnings import warn 1159 if args is None: 1160 args = [] 1161 old_cwd = os.getcwd() 1162 if cwd is not None: 1163 os.chdir(cwd) 1164 executable = venv_executable(venv=venv) 1165 venv_path = (VIRTENV_RESOURCES_PATH / venv) if venv is not None else None 1166 env_dict = kw.get('env', os.environ).copy() 1167 if venv_path is not None: 1168 env_dict.update({'VIRTUAL_ENV': venv_path.as_posix()}) 1169 command = [executable, '-m', str(package_name)] + [str(a) for a in args] 1170 import traceback 1171 if debug: 1172 print(command, file=sys.stderr) 1173 try: 1174 to_return = run_process( 1175 command, 1176 foreground = foreground, 1177 as_proc = as_proc, 1178 capture_output = capture_output, 1179 **kw 1180 ) 1181 except Exception as e: 1182 msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}" 1183 warn(msg, color=False) 1184 stdout, stderr = ( 1185 (None, None) 1186 if not capture_output 1187 else (subprocess.PIPE, subprocess.PIPE) 1188 ) 1189 proc = subprocess.Popen( 1190 command, 1191 stdout = stdout, 1192 stderr = stderr, 1193 env = env_dict, 1194 ) 1195 to_return = proc if as_proc else proc.wait() 1196 except KeyboardInterrupt: 1197 to_return = 1 if not as_proc else None 1198 os.chdir(old_cwd) 1199 return to_return
Runs an installed python package.
E.g. Translates to /usr/bin/python -m [package]
Parameters
- package_name (str): The Python module to be executed.
- args (Optional[List[str]], default None):
Additional command line arguments to be appended after
-m [package]
. - venv (Optional[str], default 'mrsm'): If specified, execute the Python interpreter from a virtual environment.
- cwd (Optional[str], default None):
If specified, change directories before starting the process.
Defaults to
None
. - as_proc (bool, default False):
If
True
, return asubprocess.Popen
object. - capture_output (bool, default False):
If
as_proc
isTrue
, capture stdout and stderr. - foreground (bool, default False):
If
True
, start the subprocess as a foreground process. Defaults toFalse
. - kw (Any):
Additional keyword arguments to pass to
meerschaum.utils.process.run_process()
and by extensionsubprocess.Popen()
.
Returns
- Either a return code integer or a
subprocess.Popen
object - (or
None
if aKeyboardInterrupt
occurs and as_proc isTrue
).
1202def attempt_import( 1203 *names: str, 1204 lazy: bool = True, 1205 warn: bool = True, 1206 install: bool = True, 1207 venv: Optional[str] = 'mrsm', 1208 precheck: bool = True, 1209 split: bool = True, 1210 check_update: bool = False, 1211 check_pypi: bool = False, 1212 check_is_installed: bool = True, 1213 allow_outside_venv: bool = True, 1214 color: bool = True, 1215 debug: bool = False 1216 ) -> Any: 1217 """ 1218 Raise a warning if packages are not installed; otherwise import and return modules. 1219 If `lazy` is `True`, return lazy-imported modules. 1220 1221 Returns tuple of modules if multiple names are provided, else returns one module. 1222 1223 Parameters 1224 ---------- 1225 names: List[str] 1226 The packages to be imported. 1227 1228 lazy: bool, default True 1229 If `True`, lazily load packages. 1230 1231 warn: bool, default True 1232 If `True`, raise a warning if a package cannot be imported. 1233 1234 install: bool, default True 1235 If `True`, attempt to install a missing package into the designated virtual environment. 1236 If `check_update` is True, install updates if available. 1237 1238 venv: Optional[str], default 'mrsm' 1239 The virtual environment in which to search for packages and to install packages into. 1240 1241 precheck: bool, default True 1242 If `True`, attempt to find module before importing (necessary for checking if modules exist 1243 and retaining lazy imports), otherwise assume lazy is `False`. 1244 1245 split: bool, default True 1246 If `True`, split packages' names on `'.'`. 1247 1248 check_update: bool, default False 1249 If `True` and `install` is `True`, install updates if the required minimum version 1250 does not match. 1251 1252 check_pypi: bool, default False 1253 If `True` and `check_update` is `True`, check PyPI when determining whether 1254 an update is required. 1255 1256 check_is_installed: bool, default True 1257 If `True`, check if the package is contained in the virtual environment. 1258 1259 allow_outside_venv: bool, default True 1260 If `True`, search outside of the specified virtual environment 1261 if the package cannot be found. 1262 Setting to `False` will reinstall the package into a virtual environment, even if it 1263 is installed outside. 1264 1265 color: bool, default True 1266 If `False`, do not print ANSI colors. 1267 1268 Returns 1269 ------- 1270 The specified modules. If they're not available and `install` is `True`, it will first 1271 download them into a virtual environment and return the modules. 1272 1273 Examples 1274 -------- 1275 >>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy') 1276 >>> pandas = attempt_import('pandas') 1277 1278 """ 1279 1280 import importlib.util 1281 1282 ### to prevent recursion, check if parent Meerschaum package is being imported 1283 if names == ('meerschaum',): 1284 return _import_module('meerschaum') 1285 1286 if venv == 'mrsm' and _import_hook_venv is not None: 1287 if debug: 1288 print(f"Import hook for virtual environment '{_import_hook_venv}' is active.") 1289 venv = _import_hook_venv 1290 1291 _warnings = _import_module('meerschaum.utils.warnings') 1292 warn_function = _warnings.warn 1293 1294 def do_import(_name: str, **kw) -> Union['ModuleType', None]: 1295 with Venv(venv=venv, debug=debug): 1296 ### determine the import method (lazy vs normal) 1297 from meerschaum.utils.misc import filter_keywords 1298 import_method = ( 1299 _import_module if not lazy 1300 else lazy_import 1301 ) 1302 try: 1303 mod = import_method(_name, **(filter_keywords(import_method, **kw))) 1304 except Exception as e: 1305 if warn: 1306 import traceback 1307 traceback.print_exception(type(e), e, e.__traceback__) 1308 warn_function( 1309 f"Failed to import module '{_name}'.\nException:\n{e}", 1310 ImportWarning, 1311 stacklevel = (5 if lazy else 4), 1312 color = False, 1313 ) 1314 mod = None 1315 return mod 1316 1317 modules = [] 1318 for name in names: 1319 ### Check if package is a declared dependency. 1320 root_name = name.split('.')[0] if split else name 1321 install_name = _import_to_install_name(root_name) 1322 1323 if install_name is None: 1324 install_name = root_name 1325 if warn and root_name != 'plugins': 1326 warn_function( 1327 f"Package '{root_name}' is not declared in meerschaum.utils.packages.", 1328 ImportWarning, 1329 stacklevel = 3, 1330 color = False 1331 ) 1332 1333 ### Determine if the package exists. 1334 if precheck is False: 1335 found_module = ( 1336 do_import( 1337 name, debug=debug, warn=False, venv=venv, color=color, 1338 check_update=False, check_pypi=False, split=split, 1339 ) is not None 1340 ) 1341 else: 1342 if check_is_installed: 1343 with _locks['_is_installed_first_check']: 1344 if not _is_installed_first_check.get(name, False): 1345 package_is_installed = is_installed( 1346 name, 1347 venv = venv, 1348 split = split, 1349 allow_outside_venv = allow_outside_venv, 1350 debug = debug, 1351 ) 1352 _is_installed_first_check[name] = package_is_installed 1353 else: 1354 package_is_installed = _is_installed_first_check[name] 1355 else: 1356 package_is_installed = _is_installed_first_check.get( 1357 name, 1358 venv_contains_package(name, venv=venv, split=split, debug=debug) 1359 ) 1360 found_module = package_is_installed 1361 1362 if not found_module: 1363 if install: 1364 if not pip_install( 1365 install_name, 1366 venv = venv, 1367 split = False, 1368 check_update = check_update, 1369 color = color, 1370 debug = debug 1371 ) and warn: 1372 warn_function( 1373 f"Failed to install '{install_name}'.", 1374 ImportWarning, 1375 stacklevel = 3, 1376 color = False, 1377 ) 1378 elif warn: 1379 ### Raise a warning if we can't find the package and install = False. 1380 warn_function( 1381 (f"\n\nMissing package '{name}' from virtual environment '{venv}'; " 1382 + "some features will not work correctly." 1383 + f"\n\nSet install=True when calling attempt_import.\n"), 1384 ImportWarning, 1385 stacklevel = 3, 1386 color = False, 1387 ) 1388 1389 ### Do the import. Will be lazy if lazy=True. 1390 m = do_import( 1391 name, debug=debug, warn=warn, venv=venv, color=color, 1392 check_update=check_update, check_pypi=check_pypi, install=install, split=split, 1393 ) 1394 modules.append(m) 1395 1396 modules = tuple(modules) 1397 if len(modules) == 1: 1398 return modules[0] 1399 return modules
Raise a warning if packages are not installed; otherwise import and return modules.
If lazy
is True
, return lazy-imported modules.
Returns tuple of modules if multiple names are provided, else returns one module.
Parameters
- names (List[str]): The packages to be imported.
- lazy (bool, default True):
If
True
, lazily load packages. - warn (bool, default True):
If
True
, raise a warning if a package cannot be imported. - install (bool, default True):
If
True
, attempt to install a missing package into the designated virtual environment. Ifcheck_update
is True, install updates if available. - venv (Optional[str], default 'mrsm'): The virtual environment in which to search for packages and to install packages into.
- precheck (bool, default True):
If
True
, attempt to find module before importing (necessary for checking if modules exist and retaining lazy imports), otherwise assume lazy isFalse
. - split (bool, default True):
If
True
, split packages' names on'.'
. - check_update (bool, default False):
If
True
andinstall
isTrue
, install updates if the required minimum version does not match. - check_pypi (bool, default False):
If
True
andcheck_update
isTrue
, check PyPI when determining whether an update is required. - check_is_installed (bool, default True):
If
True
, check if the package is contained in the virtual environment. - allow_outside_venv (bool, default True):
If
True
, search outside of the specified virtual environment if the package cannot be found. Setting toFalse
will reinstall the package into a virtual environment, even if it is installed outside. - color (bool, default True):
If
False
, do not print ANSI colors.
Returns
- The specified modules. If they're not available and
install
isTrue
, it will first - download them into a virtual environment and return the modules.
Examples
>>> pandas, sqlalchemy = attempt_import('pandas', 'sqlalchemy')
>>> pandas = attempt_import('pandas')
1402def lazy_import( 1403 name: str, 1404 local_name: str = None, 1405 **kw 1406 ) -> meerschaum.utils.packages.lazy_loader.LazyLoader: 1407 """ 1408 Lazily import a package. 1409 """ 1410 from meerschaum.utils.packages.lazy_loader import LazyLoader 1411 if local_name is None: 1412 local_name = name 1413 return LazyLoader( 1414 local_name, 1415 globals(), 1416 name, 1417 **kw 1418 )
Lazily import a package.
1421def pandas_name() -> str: 1422 """ 1423 Return the configured name for `pandas`. 1424 1425 Below are the expected possible values: 1426 1427 - 'pandas' 1428 - 'modin.pandas' 1429 - 'dask.dataframe' 1430 1431 """ 1432 from meerschaum.config import get_config 1433 pandas_module_name = get_config('system', 'connectors', 'all', 'pandas', patch=True) 1434 if pandas_module_name == 'modin': 1435 pandas_module_name = 'modin.pandas' 1436 elif pandas_module_name == 'dask': 1437 pandas_module_name = 'dask.dataframe' 1438 1439 return pandas_module_name
Return the configured name for pandas
.
Below are the expected possible values:
- 'pandas'
- 'modin.pandas'
- 'dask.dataframe'
1443def import_pandas( 1444 debug: bool = False, 1445 lazy: bool = False, 1446 **kw 1447 ) -> 'ModuleType': 1448 """ 1449 Quality-of-life function to attempt to import the configured version of `pandas`. 1450 """ 1451 import sys 1452 pandas_module_name = pandas_name() 1453 global emitted_pandas_warning 1454 1455 if pandas_module_name != 'pandas': 1456 with _locks['emitted_pandas_warning']: 1457 if not emitted_pandas_warning: 1458 from meerschaum.utils.warnings import warn 1459 emitted_pandas_warning = True 1460 warn( 1461 ( 1462 "You are using an alternative Pandas implementation " 1463 + f"'{pandas_module_name}'" 1464 + "\n Features may not work as expected." 1465 ), 1466 stack = False, 1467 ) 1468 1469 pytz = attempt_import('pytz', debug=debug, lazy=False, **kw) 1470 pandas = attempt_import('pandas', debug=debug, lazy=False, **kw) 1471 pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw) 1472 return pd
Quality-of-life function to attempt to import the configured version of pandas
.
1475def import_rich( 1476 lazy: bool = True, 1477 debug: bool = False, 1478 **kw : Any 1479 ) -> 'ModuleType': 1480 """ 1481 Quality of life function for importing `rich`. 1482 """ 1483 from meerschaum.utils.formatting import ANSI, UNICODE 1484 if not ANSI and not UNICODE: 1485 return None 1486 1487 ## need typing_extensions for `from rich import box` 1488 typing_extensions = attempt_import( 1489 'typing_extensions', lazy=False, debug=debug 1490 ) 1491 pygments = attempt_import( 1492 'pygments', lazy=False, 1493 ) 1494 rich = attempt_import( 1495 'rich', lazy=lazy, 1496 **kw 1497 ) 1498 return rich
Quality of life function for importing rich
.
1512def import_dcc(warn=False, **kw) -> 'ModuleType': 1513 """ 1514 Import Dash Core Components (`dcc`). 1515 """ 1516 return ( 1517 attempt_import('dash_core_components', warn=warn, **kw) 1518 if _dash_less_than_2(warn=warn, **kw) else attempt_import('dash.dcc', warn=warn, **kw) 1519 )
Import Dash Core Components (dcc
).
1522def import_html(warn=False, **kw) -> 'ModuleType': 1523 """ 1524 Import Dash HTML Components (`html`). 1525 """ 1526 return ( 1527 attempt_import('dash_html_components', warn=warn, **kw) 1528 if _dash_less_than_2(warn=warn, **kw) 1529 else attempt_import('dash.html', warn=warn, **kw) 1530 )
Import Dash HTML Components (html
).
1533def get_modules_from_package( 1534 package: 'package', 1535 names: bool = False, 1536 recursive: bool = False, 1537 lazy: bool = False, 1538 modules_venvs: bool = False, 1539 debug: bool = False 1540 ): 1541 """ 1542 Find and import all modules in a package. 1543 1544 Returns 1545 ------- 1546 Either list of modules or tuple of lists. 1547 """ 1548 from os.path import dirname, join, isfile, isdir, basename 1549 import glob 1550 1551 pattern = '*' if recursive else '*.py' 1552 package_path = dirname(package.__file__ or package.__path__[0]) 1553 module_names = glob.glob(join(package_path, pattern), recursive=recursive) 1554 _all = [ 1555 basename(f)[:-3] if isfile(f) else basename(f) 1556 for f in module_names 1557 if ((isfile(f) and f.endswith('.py')) or isdir(f)) 1558 and not f.endswith('__init__.py') 1559 and not f.endswith('__pycache__') 1560 ] 1561 1562 if debug: 1563 from meerschaum.utils.debug import dprint 1564 dprint(str(_all)) 1565 modules = [] 1566 for module_name in [package.__name__ + "." + mod_name for mod_name in _all]: 1567 ### there's probably a better way than a try: catch but it'll do for now 1568 try: 1569 ### if specified, activate the module's virtual environment before importing. 1570 ### NOTE: this only considers the filename, so two modules from different packages 1571 ### may end up sharing virtual environments. 1572 if modules_venvs: 1573 activate_venv(module_name.split('.')[-1], debug=debug) 1574 m = lazy_import(module_name, debug=debug) if lazy else _import_module(module_name) 1575 modules.append(m) 1576 except Exception as e: 1577 if debug: 1578 dprint(str(e)) 1579 finally: 1580 if modules_venvs: 1581 deactivate_venv(module_name.split('.')[-1], debug=debug) 1582 if names: 1583 return _all, modules 1584 1585 return modules
Find and import all modules in a package.
Returns
- Either list of modules or tuple of lists.
1588def import_children( 1589 package: Optional['ModuleType'] = None, 1590 package_name: Optional[str] = None, 1591 types : Optional[List[str]] = None, 1592 lazy: bool = True, 1593 recursive: bool = False, 1594 debug: bool = False 1595 ) -> List['ModuleType']: 1596 """ 1597 Import all functions in a package to its `__init__`. 1598 1599 Parameters 1600 ---------- 1601 package: Optional[ModuleType], default None 1602 Package to import its functions into. 1603 If `None` (default), use parent. 1604 1605 package_name: Optional[str], default None 1606 Name of package to import its functions into 1607 If None (default), use parent. 1608 1609 types: Optional[List[str]], default None 1610 Types of members to return. 1611 Defaults are `['method', 'builtin', 'class', 'function', 'package', 'module']` 1612 1613 Returns 1614 ------- 1615 A list of modules. 1616 """ 1617 import sys, inspect 1618 1619 if types is None: 1620 types = ['method', 'builtin', 'function', 'class', 'module'] 1621 1622 ### if package_name and package are None, use parent 1623 if package is None and package_name is None: 1624 package_name = inspect.stack()[1][0].f_globals['__name__'] 1625 1626 ### populate package or package_name from other other 1627 if package is None: 1628 package = sys.modules[package_name] 1629 elif package_name is None: 1630 package_name = package.__name__ 1631 1632 ### Set attributes in sys module version of package. 1633 ### Kinda like setting a dictionary 1634 ### functions[name] = func 1635 modules = get_modules_from_package(package, recursive=recursive, lazy=lazy, debug=debug) 1636 _all, members = [], [] 1637 objects = [] 1638 for module in modules: 1639 _objects = [] 1640 for ob in inspect.getmembers(module): 1641 for t in types: 1642 ### ob is a tuple of (name, object) 1643 if getattr(inspect, 'is' + t)(ob[1]): 1644 _objects.append(ob) 1645 1646 if 'module' in types: 1647 _objects.append((module.__name__.split('.')[0], module)) 1648 objects += _objects 1649 for ob in objects: 1650 setattr(sys.modules[package_name], ob[0], ob[1]) 1651 _all.append(ob[0]) 1652 members.append(ob[1]) 1653 1654 if debug: 1655 from meerschaum.utils.debug import dprint 1656 dprint(str(_all)) 1657 ### set __all__ for import * 1658 setattr(sys.modules[package_name], '__all__', _all) 1659 return members
Import all functions in a package to its __init__
.
Parameters
- package (Optional[ModuleType], default None):
Package to import its functions into.
If
None
(default), use parent. - package_name (Optional[str], default None): Name of package to import its functions into If None (default), use parent.
- types (Optional[List[str]], default None):
Types of members to return.
Defaults are
['method', 'builtin', 'class', 'function', 'package', 'module']
Returns
- A list of modules.
1663def reload_package( 1664 package: str, 1665 skip_submodules: Optional[List[str]] = None, 1666 lazy: bool = False, 1667 debug: bool = False, 1668 **kw: Any 1669 ): 1670 """ 1671 Recursively load a package's subpackages, even if they were not previously loaded. 1672 """ 1673 import sys 1674 if isinstance(package, str): 1675 package_name = package 1676 else: 1677 try: 1678 package_name = package.__name__ 1679 except Exception as e: 1680 package_name = str(package) 1681 1682 skip_submodules = skip_submodules or [] 1683 if 'meerschaum.utils.packages' not in skip_submodules: 1684 skip_submodules.append('meerschaum.utils.packages') 1685 def safeimport(): 1686 subs = [ 1687 m for m in sys.modules 1688 if m.startswith(package_name + '.') 1689 ] 1690 subs_to_skip = [] 1691 for skip_mod in skip_submodules: 1692 for mod in subs: 1693 if mod.startswith(skip_mod): 1694 subs_to_skip.append(mod) 1695 continue 1696 1697 subs = [m for m in subs if m not in subs_to_skip] 1698 for module_name in subs: 1699 _reload_module_cache[module_name] = sys.modules.pop(module_name, None) 1700 if not subs_to_skip: 1701 _reload_module_cache[package_name] = sys.modules.pop(package_name, None) 1702 1703 return _import_module(package_name) 1704 1705 return safeimport()
Recursively load a package's subpackages, even if they were not previously loaded.
1708def reload_meerschaum(debug: bool = False) -> SuccessTuple: 1709 """ 1710 Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration. 1711 """ 1712 reload_package( 1713 'meerschaum', 1714 skip_submodules = [ 1715 'meerschaum._internal.shell', 1716 'meerschaum.utils.pool', 1717 ] 1718 ) 1719 1720 from meerschaum.plugins import reload_plugins 1721 from meerschaum._internal.shell.Shell import _insert_shell_actions 1722 reload_plugins(debug=debug) 1723 _insert_shell_actions() 1724 return True, "Success"
Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
1727def is_installed( 1728 import_name: str, 1729 venv: Optional[str] = 'mrsm', 1730 split: bool = True, 1731 allow_outside_venv: bool = True, 1732 debug: bool = False, 1733 ) -> bool: 1734 """ 1735 Check whether a package is installed. 1736 1737 Parameters 1738 ---------- 1739 import_name: str 1740 The import name of the module. 1741 1742 venv: Optional[str], default 'mrsm' 1743 The venv in which to search for the module. 1744 1745 split: bool, default True 1746 If `True`, split on periods to determine the root module name. 1747 1748 allow_outside_venv: bool, default True 1749 If `True`, search outside of the specified virtual environment 1750 if the package cannot be found. 1751 """ 1752 if debug: 1753 from meerschaum.utils.debug import dprint 1754 root_name = import_name.split('.')[0] if split else import_name 1755 import importlib.util 1756 with Venv(venv, debug=debug): 1757 try: 1758 spec_path = pathlib.Path( 1759 get_module_path(root_name, venv=venv, debug=debug) 1760 or 1761 ( 1762 importlib.util.find_spec(root_name).origin 1763 if venv is not None and allow_outside_venv 1764 else None 1765 ) 1766 ) 1767 except (ModuleNotFoundError, ValueError, AttributeError, TypeError) as e: 1768 spec_path = None 1769 1770 found = ( 1771 not need_update( 1772 None, import_name = root_name, 1773 _run_determine_version = False, 1774 check_pypi = False, 1775 version = determine_version( 1776 spec_path, venv=venv, debug=debug, import_name=root_name 1777 ), 1778 debug = debug, 1779 ) 1780 ) if spec_path is not None else False 1781 1782 return found
Check whether a package is installed.
Parameters
- import_name (str): The import name of the module.
- venv (Optional[str], default 'mrsm'): The venv in which to search for the module.
- split (bool, default True):
If
True
, split on periods to determine the root module name. - allow_outside_venv (bool, default True):
If
True
, search outside of the specified virtual environment if the package cannot be found.
1785def venv_contains_package( 1786 import_name: str, 1787 venv: Optional[str] = 'mrsm', 1788 split: bool = True, 1789 debug: bool = False, 1790 ) -> bool: 1791 """ 1792 Search the contents of a virtual environment for a package. 1793 """ 1794 import site 1795 import pathlib 1796 root_name = import_name.split('.')[0] if split else import_name 1797 return get_module_path(root_name, venv=venv, debug=debug) is not None
Search the contents of a virtual environment for a package.
1800def package_venv(package: 'ModuleType') -> Union[str, None]: 1801 """ 1802 Inspect a package and return the virtual environment in which it presides. 1803 """ 1804 import os 1805 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 1806 if str(VIRTENV_RESOURCES_PATH) not in package.__file__: 1807 return None 1808 return package.__file__.split(str(VIRTENV_RESOURCES_PATH))[1].split(os.path.sep)[1]
Inspect a package and return the virtual environment in which it presides.
1811def ensure_readline() -> 'ModuleType': 1812 """Make sure that the `readline` package is able to be imported.""" 1813 import sys 1814 try: 1815 import readline 1816 except ImportError: 1817 readline = None 1818 1819 if readline is None: 1820 import platform 1821 rl_name = "gnureadline" if platform.system() != 'Windows' else "pyreadline3" 1822 try: 1823 rl = attempt_import( 1824 rl_name, 1825 lazy = False, 1826 install = True, 1827 venv = None, 1828 warn = False, 1829 ) 1830 except (ImportError, ModuleNotFoundError): 1831 if not pip_install(rl_name, args=['--upgrade', '--ignore-installed'], venv=None): 1832 print(f"Unable to import {rl_name}!", file=sys.stderr) 1833 sys.exit(1) 1834 1835 sys.modules['readline'] = readline 1836 return readline
Make sure that the readline
package is able to be imported.