meerschaum.utils.venv
Manage virtual environments.
1#! /usr/bin/env python3 2# -*- coding: utf-8 -*- 3# vim:fenc=utf-8 4 5""" 6Manage virtual environments. 7""" 8 9from __future__ import annotations 10 11import sys 12import pathlib 13 14from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple 15from meerschaum.utils.threading import RLock, get_ident 16 17__all__ = sorted([ 18 'activate_venv', 'deactivate_venv', 'init_venv', 19 'inside_venv', 'is_venv_active', 'venv_exec', 20 'venv_executable', 'venv_exists', 'venv_target_path', 21 'Venv', 'get_venvs', 'verify_venv', 'get_module_venv', 22]) 23__pdoc__ = {'Venv': True} 24 25LOCKS = { 26 'sys.path': RLock(), 27 'active_venvs': RLock(), 28 'venvs_active_counts': RLock(), 29} 30 31active_venvs = set() 32active_venvs_counts: Dict[str, int] = {} 33active_venvs_order: List[Optional[str]] = [] 34threads_active_venvs: Dict[int, 'set[str]'] = {} 35CREATE_NEW_PROCESS_GROUP = 0x00000200 36 37 38def activate_venv( 39 venv: Optional[str] = 'mrsm', 40 color: bool = True, 41 force: bool = False, 42 init_if_not_exists: bool = True, 43 debug: bool = False, 44 **kw 45) -> bool: 46 """ 47 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 48 49 Parameters 50 ---------- 51 venv: Optional[str], default 'mrsm' 52 The virtual environment to activate. 53 54 color: bool, default True 55 If `True`, include color in debug text. 56 57 init_if_not_exists: bool, default True 58 If `True`, create the virtual environment if it does not exist. 59 60 force: bool, default False 61 If `True`, do not exit early even if the venv is currently active. 62 63 debug: bool, default False 64 Verbosity toggle. 65 66 Returns 67 ------- 68 A bool indicating whether the virtual environment was successfully activated. 69 70 """ 71 thread_id = get_ident() 72 if active_venvs_order and active_venvs_order[0] == venv: 73 if not force: 74 return True 75 76 import sys 77 import os 78 if venv is not None and init_if_not_exists: 79 init_venv(venv=venv, debug=debug) 80 81 with LOCKS['active_venvs']: 82 if thread_id not in threads_active_venvs: 83 threads_active_venvs[thread_id] = {} 84 active_venvs.add(venv) 85 if venv not in threads_active_venvs[thread_id]: 86 threads_active_venvs[thread_id][venv] = 1 87 else: 88 threads_active_venvs[thread_id][venv] += 1 89 90 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 91 if not target_path.exists() and venv is not None and init_if_not_exists: 92 init_venv(venv=venv, force=True, debug=debug) 93 94 if not target_path.exists(): 95 if init_if_not_exists: 96 return False 97 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 98 99 target = target_path.as_posix() 100 101 if venv in active_venvs_order: 102 try: 103 sys.path.remove(target) 104 except Exception: 105 pass 106 try: 107 active_venvs_order.remove(venv) 108 except Exception: 109 pass 110 111 if venv is not None: 112 sys.path.insert(0, target) 113 else: 114 if sys.path and sys.path[0] in (os.getcwd(), ''): 115 sys.path.insert(1, target) 116 else: 117 sys.path.insert(0, target) 118 try: 119 active_venvs_order.insert(0, venv) 120 except Exception: 121 pass 122 123 return True 124 125 126def deactivate_venv( 127 venv: str = 'mrsm', 128 color: bool = True, 129 debug: bool = False, 130 previously_active_venvs: Union['set[str]', List[str], None] = None, 131 force: bool = False, 132 **kw 133) -> bool: 134 """ 135 Remove a virtual environment from `sys.path` (if it's been activated). 136 137 Parameters 138 ---------- 139 venv: str, default 'mrsm' 140 The virtual environment to deactivate. 141 142 color: bool, default True 143 If `True`, include color in debug text. 144 145 debug: bool, default False 146 Verbosity toggle. 147 148 previously_active_venvs: Union[Set[str], List[str], None] 149 If provided, skip deactivating if a virtual environment is in this iterable. 150 151 force: bool, default False 152 If `True`, forcibly deactivate the virtual environment. 153 This may cause issues with other threads, so be careful! 154 155 Returns 156 ------- 157 Return a bool indicating whether the virtual environment was successfully deactivated. 158 159 """ 160 import sys 161 thread_id = get_ident() 162 if venv is None: 163 if venv in active_venvs: 164 active_venvs.remove(venv) 165 return True 166 167 if previously_active_venvs and venv in previously_active_venvs and not force: 168 return True 169 170 with LOCKS['active_venvs']: 171 if venv in threads_active_venvs.get(thread_id, {}): 172 new_count = threads_active_venvs[thread_id][venv] - 1 173 if new_count > 0 and not force: 174 threads_active_venvs[thread_id][venv] = new_count 175 return True 176 else: 177 del threads_active_venvs[thread_id][venv] 178 179 if not force: 180 for other_thread_id, other_venvs in threads_active_venvs.items(): 181 if other_thread_id == thread_id: 182 continue 183 if venv in other_venvs: 184 return True 185 else: 186 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 187 for other_thread_id in to_delete: 188 del threads_active_venvs[other_thread_id] 189 190 if venv in active_venvs: 191 active_venvs.remove(venv) 192 193 if sys.path is None: 194 return False 195 196 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 197 with LOCKS['sys.path']: 198 if target in sys.path: 199 try: 200 sys.path.remove(target) 201 except Exception: 202 pass 203 try: 204 active_venvs_order.remove(venv) 205 except Exception: 206 pass 207 208 return True 209 210 211def is_venv_active( 212 venv: str = 'mrsm', 213 color : bool = True, 214 debug: bool = False 215) -> bool: 216 """ 217 Check if a virtual environment is active. 218 219 Parameters 220 ---------- 221 venv: str, default 'mrsm' 222 The virtual environment to check. 223 224 color: bool, default True 225 If `True`, include color in debug text. 226 227 debug: bool, default False 228 Verbosity toggle. 229 230 Returns 231 ------- 232 A bool indicating whether the virtual environment `venv` is active. 233 234 """ 235 return venv in active_venvs 236 237 238verified_venvs = set() 239def verify_venv( 240 venv: str, 241 debug: bool = False, 242) -> None: 243 """ 244 Verify that the virtual environment matches the expected state. 245 """ 246 import platform 247 import os 248 import shutil 249 import sys 250 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 251 from meerschaum.utils.process import run_process 252 from meerschaum.utils.misc import make_symlink, is_symlink 253 from meerschaum.utils.warnings import warn 254 255 venv_path = VIRTENV_RESOURCES_PATH / venv 256 bin_path = venv_path / ( 257 'bin' if platform.system() != 'Windows' else "Scripts" 258 ) 259 current_python_versioned_name = ( 260 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 261 + ('' if platform.system() != 'Windows' else '.exe') 262 ) 263 264 if not (bin_path / current_python_versioned_name).exists(): 265 init_venv(venv, verify=False, force=True, debug=debug) 266 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 267 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 268 if not current_python_in_venv_path.exists(): 269 if is_symlink(current_python_in_venv_path): 270 try: 271 current_python_in_venv_path.unlink() 272 except Exception as e: 273 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 274 try: 275 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 276 except Exception: 277 print( 278 f"Unable to create symlink {current_python_in_venv_path} " 279 + f"to {current_python_in_sys_path}." 280 ) 281 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 282 else: 283 files_to_inspect = [current_python_versioned_name] 284 285 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 286 """ 287 Return the version for the python binary at the given path. 288 """ 289 try: 290 ### It might be a broken symlink, so skip on errors. 291 if debug: 292 print(f"Getting python version for {python_path}") 293 proc = run_process( 294 [str(python_path), '-V'], 295 as_proc=True, 296 capture_output=True, 297 ) 298 stdout, stderr = proc.communicate(timeout=1.0) 299 except Exception as e: 300 ### E.g. the symlink may be broken. 301 if is_symlink(python_path): 302 try: 303 python_path.unlink() 304 except Exception as _e: 305 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 306 return None 307 return stdout.decode('utf-8').strip().replace('Python ', '') 308 309 ### Ensure the versions are symlinked correctly. 310 for filename in files_to_inspect: 311 if not filename.startswith('python'): 312 continue 313 python_path = bin_path / filename 314 version = get_python_version(python_path) 315 if version is None: 316 continue 317 try: 318 major_version = version.split('.', maxsplit=1)[0] 319 minor_version = version.split('.', maxsplit=2)[1] 320 except IndexError: 321 return 322 python_versioned_name = ( 323 'python' + major_version + '.' + minor_version 324 + ('' if platform.system() != 'Windows' else '.exe') 325 ) 326 327 ### E.g. python3.10 actually links to Python 3.10. 328 if filename == python_versioned_name: 329 try: 330 real_path = pathlib.Path(os.path.realpath(python_path)) 331 real_path_exists = real_path.exists() 332 except Exception: 333 real_path_exists = False 334 335 if not real_path_exists: 336 try: 337 python_path.unlink() 338 except Exception: 339 pass 340 init_venv(venv, verify=False, force=True, debug=debug) 341 if not python_path.exists(): 342 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 343 344 if python_path == real_path: 345 continue 346 347 try: 348 python_path.unlink() 349 except Exception: 350 pass 351 success, msg = make_symlink(real_path, python_path) 352 if not success: 353 warn(msg, color=False) 354 continue 355 356 python_versioned_path = bin_path / python_versioned_name 357 if python_versioned_path.exists(): 358 ### Avoid circular symlinks. 359 if get_python_version(python_versioned_path) == version: 360 continue 361 python_versioned_path.unlink() 362 shutil.move(python_path, python_versioned_path) 363 364 365tried_virtualenv = False 366def init_venv( 367 venv: str = 'mrsm', 368 verify: bool = True, 369 force: bool = False, 370 debug: bool = False, 371) -> bool: 372 """ 373 Initialize the virtual environment. 374 375 Parameters 376 ---------- 377 venv: str, default 'mrsm' 378 The name of the virtual environment to create. 379 380 verify: bool, default True 381 If `True`, verify that the virtual environment is in the expected state. 382 383 force: bool, default False 384 If `True`, recreate the virtual environment, even if already initalized. 385 386 Returns 387 ------- 388 A `bool` indicating success. 389 """ 390 if not force and venv in verified_venvs: 391 return True 392 if not force and venv_exists(venv, debug=debug): 393 if verify: 394 verify_venv(venv, debug=debug) 395 verified_venvs.add(venv) 396 return True 397 398 import io 399 from contextlib import redirect_stdout 400 import sys 401 import platform 402 import os 403 import shutil 404 import time 405 406 from meerschaum._internal.static import STATIC_CONFIG 407 from meerschaum.config._paths import ( 408 VIRTENV_RESOURCES_PATH, 409 VENVS_CACHE_RESOURCES_PATH, 410 ) 411 from meerschaum.utils.packages import is_uv_enabled 412 413 venv_path = VIRTENV_RESOURCES_PATH / venv 414 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 415 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 416 lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 417 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 418 419 def update_lock(active: bool): 420 try: 421 if not active: 422 if debug: 423 print(f"Releasing lock: '{lock_path}'") 424 lock_path.unlink() 425 else: 426 if debug: 427 print(f"Acquiring lock: '{lock_path}'") 428 lock_path.touch() 429 except Exception: 430 pass 431 432 def wait_for_lock(): 433 if platform.system() == 'Windows': 434 return 435 max_lock_seconds = 30.0 436 sleep_message_seconds = 5.0 437 step_sleep_seconds = 0.1 438 init_venv_check_start = time.perf_counter() 439 last_print = init_venv_check_start 440 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 441 if not lock_path.exists(): 442 break 443 444 now = time.perf_counter() 445 if debug or (now - last_print) > sleep_message_seconds: 446 print(f"Lock exists for venv '{venv}', sleeping...") 447 last_print = now 448 time.sleep(step_sleep_seconds) 449 update_lock(False) 450 451 if ( 452 not force 453 and venv == 'mrsm' 454 and os.environ.get(work_dir_env_var, None) is not None 455 and docker_home_venv_path.exists() 456 ): 457 wait_for_lock() 458 shutil.move(docker_home_venv_path, venv_path) 459 if verify: 460 verify_venv(venv, debug=debug) 461 verified_venvs.add(venv) 462 return True 463 464 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 465 global tried_virtualenv 466 try: 467 import venv as _venv 468 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 469 virtualenv = None 470 except ImportError: 471 _venv = None 472 uv = None 473 virtualenv = None 474 475 _venv_success = False 476 temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv) 477 ### NOTE: Disable site-packages movement for now. 478 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 479 480 wait_for_lock() 481 update_lock(True) 482 483 if rename_vtp: 484 if debug: 485 print(f"Moving '{vtp}' to '{temp_vtp}'...") 486 shutil.move(vtp, temp_vtp) 487 488 if uv is not None: 489 _venv_success = run_python_package( 490 'uv', 491 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 492 venv=None, 493 env=_get_pip_os_env(), 494 debug=debug, 495 ) == 0 496 497 if _venv is not None and not _venv_success: 498 f = io.StringIO() 499 with redirect_stdout(f): 500 _venv_success = run_python_package( 501 'venv', 502 [venv_path.as_posix()] + ( 503 ['--symlinks'] 504 if platform.system() != 'Windows' 505 else [] 506 ), 507 venv=None, debug=debug 508 ) == 0 509 if not _venv_success: 510 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 511 if not venv_exists(venv, debug=debug): 512 _venv = None 513 if not _venv_success: 514 virtualenv = attempt_import( 515 'virtualenv', 516 venv=None, 517 lazy=False, 518 install=(not tried_virtualenv), 519 warn=False, 520 check_update=False, 521 color=False, 522 debug=debug, 523 ) 524 if virtualenv is None: 525 print( 526 "Failed to import `venv` or `virtualenv`! " 527 + "Please install `virtualenv` via pip then restart Meerschaum." 528 ) 529 if rename_vtp and temp_vtp.exists(): 530 if debug: 531 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 532 shutil.move(temp_vtp, vtp) 533 update_lock(False) 534 return False 535 536 tried_virtualenv = True 537 try: 538 python_folder = ( 539 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 540 ) 541 dist_packages_path = ( 542 VIRTENV_RESOURCES_PATH / 543 venv / 'local' / 'lib' / python_folder / 'dist-packages' 544 ) 545 local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 546 bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin' 547 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 548 if bin_path.exists(): 549 try: 550 shutil.rmtree(bin_path) 551 except Exception: 552 import traceback 553 traceback.print_exc() 554 virtualenv.cli_run([venv_path.as_posix()]) 555 if dist_packages_path.exists(): 556 vtp.mkdir(exist_ok=True, parents=True) 557 for file_path in dist_packages_path.glob('*'): 558 shutil.move(file_path, vtp) 559 shutil.rmtree(dist_packages_path) 560 # shutil.move(dist_packages_path, vtp) 561 bin_path.mkdir(exist_ok=True, parents=True) 562 for file_path in local_bin_path.glob('*'): 563 shutil.move(file_path, bin_path) 564 # shutil.move(local_bin_path, bin_path) 565 shutil.rmtree(local_bin_path) 566 567 except Exception: 568 import traceback 569 traceback.print_exc() 570 if rename_vtp and temp_vtp.exists(): 571 shutil.move(temp_vtp, vtp) 572 update_lock(False) 573 return False 574 if verify: 575 verify_venv(venv, debug=debug) 576 verified_venvs.add(venv) 577 578 if rename_vtp and temp_vtp.exists(): 579 if debug: 580 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 581 shutil.move(temp_vtp, vtp) 582 583 update_lock(False) 584 return True 585 586 587def venv_executable(venv: Optional[str] = 'mrsm') -> str: 588 """ 589 The Python interpreter executable for a given virtual environment. 590 """ 591 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 592 import sys 593 import platform 594 return ( 595 sys.executable if venv is None 596 else str( 597 VIRTENV_RESOURCES_PATH 598 / venv 599 / ( 600 'bin' if platform.system() != 'Windows' 601 else 'Scripts' 602 ) / ( 603 'python' 604 + str(sys.version_info.major) 605 + '.' 606 + str(sys.version_info.minor) 607 ) 608 ) 609 ) 610 611 612def venv_exec( 613 code: str, 614 venv: Optional[str] = 'mrsm', 615 env: Optional[Dict[str, str]] = None, 616 with_extras: bool = False, 617 as_proc: bool = False, 618 capture_output: bool = True, 619 debug: bool = False, 620) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 621 """ 622 Execute Python code in a subprocess via a virtual environment's interpeter. 623 Return `True` if the code successfully executes, `False` on failure. 624 625 Parameters 626 ---------- 627 code: str 628 The Python code to excecute. 629 630 venv: str, default 'mrsm' 631 The virtual environment to use to get the path for the Python executable. 632 If `venv` is `None`, use the default `sys.executable` path. 633 634 env: Optional[Dict[str, str]], default None 635 Optionally specify the environment variables for the subprocess. 636 Defaults to `os.environ`. 637 638 with_extras: bool, default False 639 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 640 641 as_proc: bool, default False 642 If `True`, return the `subprocess.Popen` object instead of executing. 643 644 Returns 645 ------- 646 By default, return a bool indicating success. 647 If `as_proc` is `True`, return a `subprocess.Popen` object. 648 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 649 650 """ 651 import os 652 import subprocess 653 import platform 654 from meerschaum.utils.debug import dprint 655 from meerschaum.utils.process import _child_processes 656 657 executable = venv_executable(venv=venv) 658 cmd_list = [executable, '-c', code] 659 if env is None: 660 env = os.environ 661 if debug: 662 dprint(str(cmd_list)) 663 if not with_extras and not as_proc: 664 return subprocess.call(cmd_list, env=env) == 0 665 666 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 667 group_kwargs = ( 668 { 669 'preexec_fn': os.setsid, 670 } if platform.system() != 'Windows' 671 else { 672 'creationflags': CREATE_NEW_PROCESS_GROUP, 673 } 674 ) 675 process = subprocess.Popen( 676 cmd_list, 677 stdout=stdout, 678 stderr=stderr, 679 stdin=sys.stdin, 680 env=env, 681 **group_kwargs 682 ) 683 if as_proc: 684 _child_processes.append(process) 685 return process 686 stdout, stderr = process.communicate() 687 exit_code = process.returncode 688 return exit_code, stdout, stderr 689 690 691def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 692 """ 693 Determine whether a virtual environment has been created. 694 """ 695 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 696 return target_path.exists() 697 698 699def venv_target_path( 700 venv: Union[str, None], 701 allow_nonexistent: bool = False, 702 debug: bool = False, 703) -> pathlib.Path: 704 """ 705 Return a virtual environment's site-package path. 706 707 Parameters 708 ---------- 709 venv: Union[str, None] 710 The virtual environment for which a path should be returned. 711 712 allow_nonexistent: bool, default False 713 If `True`, return a path even if it does not exist. 714 715 Returns 716 ------- 717 The `pathlib.Path` object for the virtual environment's path. 718 719 """ 720 import os 721 import sys 722 import platform 723 import site 724 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 725 from meerschaum._internal.static import STATIC_CONFIG 726 727 ### Check sys.path for a user-writable site-packages directory. 728 if venv is None: 729 730 ### Return the known value for the portable environment. 731 environment_runtime = STATIC_CONFIG['environment']['runtime'] 732 if os.environ.get(environment_runtime, None) == 'portable': 733 python_version_folder = ( 734 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 735 ) 736 executable_path = pathlib.Path(sys.executable) 737 site_packages_path = ( 738 ( 739 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 740 ) if platform.system() != 'Windows' else ( 741 executable_path.parent / 'Lib' / 'site-packages' 742 ) 743 ) 744 if not site_packages_path.exists(): 745 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 746 return site_packages_path 747 748 if not inside_venv(): 749 user_site_packages = site.getusersitepackages() 750 if user_site_packages is None: 751 raise EnvironmentError("Could not determine user site packages.") 752 753 site_path = pathlib.Path(user_site_packages) 754 if not site_path.exists(): 755 756 ### Windows does not have `os.geteuid()`. 757 if platform.system() == 'Windows' or os.geteuid() != 0: 758 site_path.mkdir(parents=True, exist_ok=True) 759 return site_path 760 761 ### Allow for dist-level paths (running as root). 762 for possible_dist in site.getsitepackages(): 763 dist_path = pathlib.Path(possible_dist) 764 if not dist_path.exists(): 765 continue 766 return dist_path 767 768 raise EnvironmentError("Could not determine the dist-packages directory.") 769 770 return site_path 771 772 venv_root_path = ( 773 (VIRTENV_RESOURCES_PATH / venv) 774 if venv is not None 775 else pathlib.Path(sys.prefix) 776 ) 777 target_path = venv_root_path 778 779 ### Ensure 'lib' or 'Lib' exists. 780 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 781 if not allow_nonexistent: 782 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 783 print(f"Failed to find lib directory for virtual environment '{venv}'.") 784 import traceback 785 traceback.print_stack() 786 sys.exit(1) 787 target_path = target_path / lib 788 789 ### Check if a 'python3.x' folder exists. 790 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 791 if target_path.exists(): 792 target_path = ( 793 (target_path / python_folder) 794 if python_folder in os.listdir(target_path) 795 else target_path 796 ) 797 else: 798 target_path = ( 799 (target_path / python_folder) 800 if platform.system() != 'Windows' 801 else target_path 802 ) 803 804 ### Ensure 'site-packages' exists. 805 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 806 target_path = target_path / 'site-packages' 807 else: 808 import traceback 809 traceback.print_stack() 810 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 811 print("This may be because you are using a different Python version.") 812 print("Try deleting the following directory and restarting Meerschaum:") 813 print(VIRTENV_RESOURCES_PATH) 814 sys.exit(1) 815 816 return target_path 817 818 819def inside_venv() -> bool: 820 """ 821 Determine whether current Python interpreter is running inside a virtual environment. 822 """ 823 import sys 824 return ( 825 hasattr(sys, 'real_prefix') or ( 826 hasattr(sys, 'base_prefix') 827 and sys.base_prefix != sys.prefix 828 ) 829 ) 830 831 832def get_venvs() -> List[str]: 833 """ 834 Return a list of all the virtual environments. 835 """ 836 import os 837 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 838 venvs = [] 839 for filename in os.listdir(VIRTENV_RESOURCES_PATH): 840 path = VIRTENV_RESOURCES_PATH / filename 841 if not path.is_dir(): 842 continue 843 if not venv_exists(filename): 844 continue 845 venvs.append(filename) 846 return venvs 847 848 849def get_module_venv(module) -> Union[str, None]: 850 """ 851 Return the virtual environment where an imported module is installed. 852 853 Parameters 854 ---------- 855 module: ModuleType 856 The imported module to inspect. 857 858 Returns 859 ------- 860 The name of a venv or `None`. 861 """ 862 from meerschaum.config.paths import VIRTENV_RESOURCES_PATH 863 module_path = pathlib.Path(module.__file__).resolve() 864 try: 865 rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH) 866 except ValueError: 867 return None 868 869 return rel_path.as_posix().split('/', maxsplit=1)[0] 870 871 872from meerschaum.utils.venv._Venv import Venv
19class Venv: 20 """ 21 Manage a virtual enviroment's activation status. 22 23 Examples 24 -------- 25 >>> from meerschaum.plugins import Plugin 26 >>> with Venv('mrsm') as venv: 27 ... import pandas 28 >>> with Venv(Plugin('noaa')) as venv: 29 ... import requests 30 >>> venv = Venv('mrsm') 31 >>> venv.activate() 32 True 33 >>> venv.deactivate() 34 True 35 >>> 36 """ 37 38 def __init__( 39 self, 40 venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm', 41 init_if_not_exists: bool = True, 42 debug: bool = False, 43 ) -> None: 44 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 45 ### For some weird threading issue, 46 ### we can't use `isinstance` here. 47 if '_Plugin' in str(type(venv)): 48 self._venv = venv.name 49 self._activate = venv.activate_venv 50 self._deactivate = venv.deactivate_venv 51 self._kwargs = {} 52 else: 53 self._venv = venv 54 self._activate = activate_venv 55 self._deactivate = deactivate_venv 56 self._kwargs = {'venv': venv} 57 self._debug = debug 58 self._init_if_not_exists = init_if_not_exists 59 ### In case someone calls `deactivate()` before `activate()`. 60 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 61 62 63 def activate(self, debug: bool = False) -> bool: 64 """ 65 Activate this virtual environment. 66 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 67 will also be activated. 68 """ 69 from meerschaum.utils.venv import active_venvs, init_venv 70 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 71 try: 72 return self._activate( 73 debug=(debug or self._debug), 74 init_if_not_exists=self._init_if_not_exists, 75 **self._kwargs 76 ) 77 except OSError as e: 78 if self._init_if_not_exists: 79 if not init_venv(self._venv, force=True): 80 raise e 81 return self._activate( 82 debug=(debug or self._debug), 83 init_if_not_exists=self._init_if_not_exists, 84 **self._kwargs 85 ) 86 87 88 def deactivate(self, debug: bool = False) -> bool: 89 """ 90 Deactivate this virtual environment. 91 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 92 will also be deactivated. 93 """ 94 return self._deactivate(debug=(debug or self._debug), **self._kwargs) 95 96 97 @property 98 def target_path(self) -> pathlib.Path: 99 """ 100 Return the target site-packages path for this virtual environment. 101 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 102 (e.g. Python 3.10 and Python 3.7). 103 """ 104 from meerschaum.utils.venv import venv_target_path 105 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug) 106 107 108 @property 109 def root_path(self) -> pathlib.Path: 110 """ 111 Return the top-level path for this virtual environment. 112 """ 113 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 114 if self._venv is None: 115 return self.target_path.parent 116 return VIRTENV_RESOURCES_PATH / self._venv 117 118 119 def __enter__(self) -> None: 120 self.activate(debug=self._debug) 121 122 123 def __exit__(self, exc_type, exc_value, exc_traceback) -> None: 124 self.deactivate(debug=self._debug) 125 126 127 def __str__(self) -> str: 128 quote = "'" if self._venv is not None else "" 129 return "Venv(" + quote + str(self._venv) + quote + ")" 130 131 132 def __repr__(self) -> str: 133 return self.__str__()
Manage a virtual enviroment's activation status.
Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
... import pandas
>>> with Venv(Plugin('noaa')) as venv:
... import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
38 def __init__( 39 self, 40 venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm', 41 init_if_not_exists: bool = True, 42 debug: bool = False, 43 ) -> None: 44 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 45 ### For some weird threading issue, 46 ### we can't use `isinstance` here. 47 if '_Plugin' in str(type(venv)): 48 self._venv = venv.name 49 self._activate = venv.activate_venv 50 self._deactivate = venv.deactivate_venv 51 self._kwargs = {} 52 else: 53 self._venv = venv 54 self._activate = activate_venv 55 self._deactivate = deactivate_venv 56 self._kwargs = {'venv': venv} 57 self._debug = debug 58 self._init_if_not_exists = init_if_not_exists 59 ### In case someone calls `deactivate()` before `activate()`. 60 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
63 def activate(self, debug: bool = False) -> bool: 64 """ 65 Activate this virtual environment. 66 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 67 will also be activated. 68 """ 69 from meerschaum.utils.venv import active_venvs, init_venv 70 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 71 try: 72 return self._activate( 73 debug=(debug or self._debug), 74 init_if_not_exists=self._init_if_not_exists, 75 **self._kwargs 76 ) 77 except OSError as e: 78 if self._init_if_not_exists: 79 if not init_venv(self._venv, force=True): 80 raise e 81 return self._activate( 82 debug=(debug or self._debug), 83 init_if_not_exists=self._init_if_not_exists, 84 **self._kwargs 85 )
Activate this virtual environment.
If a meerschaum.plugins.Plugin
was provided, its dependent virtual environments
will also be activated.
88 def deactivate(self, debug: bool = False) -> bool: 89 """ 90 Deactivate this virtual environment. 91 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 92 will also be deactivated. 93 """ 94 return self._deactivate(debug=(debug or self._debug), **self._kwargs)
Deactivate this virtual environment.
If a meerschaum.plugins.Plugin
was provided, its dependent virtual environments
will also be deactivated.
97 @property 98 def target_path(self) -> pathlib.Path: 99 """ 100 Return the target site-packages path for this virtual environment. 101 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 102 (e.g. Python 3.10 and Python 3.7). 103 """ 104 from meerschaum.utils.venv import venv_target_path 105 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
Return the target site-packages path for this virtual environment.
A meerschaum.utils.venv.Venv
may have one virtual environment per minor Python version
(e.g. Python 3.10 and Python 3.7).
108 @property 109 def root_path(self) -> pathlib.Path: 110 """ 111 Return the top-level path for this virtual environment. 112 """ 113 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 114 if self._venv is None: 115 return self.target_path.parent 116 return VIRTENV_RESOURCES_PATH / self._venv
Return the top-level path for this virtual environment.
39def activate_venv( 40 venv: Optional[str] = 'mrsm', 41 color: bool = True, 42 force: bool = False, 43 init_if_not_exists: bool = True, 44 debug: bool = False, 45 **kw 46) -> bool: 47 """ 48 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 49 50 Parameters 51 ---------- 52 venv: Optional[str], default 'mrsm' 53 The virtual environment to activate. 54 55 color: bool, default True 56 If `True`, include color in debug text. 57 58 init_if_not_exists: bool, default True 59 If `True`, create the virtual environment if it does not exist. 60 61 force: bool, default False 62 If `True`, do not exit early even if the venv is currently active. 63 64 debug: bool, default False 65 Verbosity toggle. 66 67 Returns 68 ------- 69 A bool indicating whether the virtual environment was successfully activated. 70 71 """ 72 thread_id = get_ident() 73 if active_venvs_order and active_venvs_order[0] == venv: 74 if not force: 75 return True 76 77 import sys 78 import os 79 if venv is not None and init_if_not_exists: 80 init_venv(venv=venv, debug=debug) 81 82 with LOCKS['active_venvs']: 83 if thread_id not in threads_active_venvs: 84 threads_active_venvs[thread_id] = {} 85 active_venvs.add(venv) 86 if venv not in threads_active_venvs[thread_id]: 87 threads_active_venvs[thread_id][venv] = 1 88 else: 89 threads_active_venvs[thread_id][venv] += 1 90 91 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 92 if not target_path.exists() and venv is not None and init_if_not_exists: 93 init_venv(venv=venv, force=True, debug=debug) 94 95 if not target_path.exists(): 96 if init_if_not_exists: 97 return False 98 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 99 100 target = target_path.as_posix() 101 102 if venv in active_venvs_order: 103 try: 104 sys.path.remove(target) 105 except Exception: 106 pass 107 try: 108 active_venvs_order.remove(venv) 109 except Exception: 110 pass 111 112 if venv is not None: 113 sys.path.insert(0, target) 114 else: 115 if sys.path and sys.path[0] in (os.getcwd(), ''): 116 sys.path.insert(1, target) 117 else: 118 sys.path.insert(0, target) 119 try: 120 active_venvs_order.insert(0, venv) 121 except Exception: 122 pass 123 124 return True
Create a virtual environment (if it doesn't exist) and add it to sys.path
if necessary.
Parameters
- venv (Optional[str], default 'mrsm'): The virtual environment to activate.
- color (bool, default True):
If
True
, include color in debug text. - init_if_not_exists (bool, default True):
If
True
, create the virtual environment if it does not exist. - force (bool, default False):
If
True
, do not exit early even if the venv is currently active. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment was successfully activated.
127def deactivate_venv( 128 venv: str = 'mrsm', 129 color: bool = True, 130 debug: bool = False, 131 previously_active_venvs: Union['set[str]', List[str], None] = None, 132 force: bool = False, 133 **kw 134) -> bool: 135 """ 136 Remove a virtual environment from `sys.path` (if it's been activated). 137 138 Parameters 139 ---------- 140 venv: str, default 'mrsm' 141 The virtual environment to deactivate. 142 143 color: bool, default True 144 If `True`, include color in debug text. 145 146 debug: bool, default False 147 Verbosity toggle. 148 149 previously_active_venvs: Union[Set[str], List[str], None] 150 If provided, skip deactivating if a virtual environment is in this iterable. 151 152 force: bool, default False 153 If `True`, forcibly deactivate the virtual environment. 154 This may cause issues with other threads, so be careful! 155 156 Returns 157 ------- 158 Return a bool indicating whether the virtual environment was successfully deactivated. 159 160 """ 161 import sys 162 thread_id = get_ident() 163 if venv is None: 164 if venv in active_venvs: 165 active_venvs.remove(venv) 166 return True 167 168 if previously_active_venvs and venv in previously_active_venvs and not force: 169 return True 170 171 with LOCKS['active_venvs']: 172 if venv in threads_active_venvs.get(thread_id, {}): 173 new_count = threads_active_venvs[thread_id][venv] - 1 174 if new_count > 0 and not force: 175 threads_active_venvs[thread_id][venv] = new_count 176 return True 177 else: 178 del threads_active_venvs[thread_id][venv] 179 180 if not force: 181 for other_thread_id, other_venvs in threads_active_venvs.items(): 182 if other_thread_id == thread_id: 183 continue 184 if venv in other_venvs: 185 return True 186 else: 187 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 188 for other_thread_id in to_delete: 189 del threads_active_venvs[other_thread_id] 190 191 if venv in active_venvs: 192 active_venvs.remove(venv) 193 194 if sys.path is None: 195 return False 196 197 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 198 with LOCKS['sys.path']: 199 if target in sys.path: 200 try: 201 sys.path.remove(target) 202 except Exception: 203 pass 204 try: 205 active_venvs_order.remove(venv) 206 except Exception: 207 pass 208 209 return True
Remove a virtual environment from sys.path
(if it's been activated).
Parameters
- venv (str, default 'mrsm'): The virtual environment to deactivate.
- color (bool, default True):
If
True
, include color in debug text. - debug (bool, default False): Verbosity toggle.
- previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
- force (bool, default False):
If
True
, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
- Return a bool indicating whether the virtual environment was successfully deactivated.
850def get_module_venv(module) -> Union[str, None]: 851 """ 852 Return the virtual environment where an imported module is installed. 853 854 Parameters 855 ---------- 856 module: ModuleType 857 The imported module to inspect. 858 859 Returns 860 ------- 861 The name of a venv or `None`. 862 """ 863 from meerschaum.config.paths import VIRTENV_RESOURCES_PATH 864 module_path = pathlib.Path(module.__file__).resolve() 865 try: 866 rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH) 867 except ValueError: 868 return None 869 870 return rel_path.as_posix().split('/', maxsplit=1)[0]
Return the virtual environment where an imported module is installed.
Parameters
- module (ModuleType): The imported module to inspect.
Returns
- The name of a venv or
None
.
833def get_venvs() -> List[str]: 834 """ 835 Return a list of all the virtual environments. 836 """ 837 import os 838 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 839 venvs = [] 840 for filename in os.listdir(VIRTENV_RESOURCES_PATH): 841 path = VIRTENV_RESOURCES_PATH / filename 842 if not path.is_dir(): 843 continue 844 if not venv_exists(filename): 845 continue 846 venvs.append(filename) 847 return venvs
Return a list of all the virtual environments.
367def init_venv( 368 venv: str = 'mrsm', 369 verify: bool = True, 370 force: bool = False, 371 debug: bool = False, 372) -> bool: 373 """ 374 Initialize the virtual environment. 375 376 Parameters 377 ---------- 378 venv: str, default 'mrsm' 379 The name of the virtual environment to create. 380 381 verify: bool, default True 382 If `True`, verify that the virtual environment is in the expected state. 383 384 force: bool, default False 385 If `True`, recreate the virtual environment, even if already initalized. 386 387 Returns 388 ------- 389 A `bool` indicating success. 390 """ 391 if not force and venv in verified_venvs: 392 return True 393 if not force and venv_exists(venv, debug=debug): 394 if verify: 395 verify_venv(venv, debug=debug) 396 verified_venvs.add(venv) 397 return True 398 399 import io 400 from contextlib import redirect_stdout 401 import sys 402 import platform 403 import os 404 import shutil 405 import time 406 407 from meerschaum._internal.static import STATIC_CONFIG 408 from meerschaum.config._paths import ( 409 VIRTENV_RESOURCES_PATH, 410 VENVS_CACHE_RESOURCES_PATH, 411 ) 412 from meerschaum.utils.packages import is_uv_enabled 413 414 venv_path = VIRTENV_RESOURCES_PATH / venv 415 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 416 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 417 lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 418 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 419 420 def update_lock(active: bool): 421 try: 422 if not active: 423 if debug: 424 print(f"Releasing lock: '{lock_path}'") 425 lock_path.unlink() 426 else: 427 if debug: 428 print(f"Acquiring lock: '{lock_path}'") 429 lock_path.touch() 430 except Exception: 431 pass 432 433 def wait_for_lock(): 434 if platform.system() == 'Windows': 435 return 436 max_lock_seconds = 30.0 437 sleep_message_seconds = 5.0 438 step_sleep_seconds = 0.1 439 init_venv_check_start = time.perf_counter() 440 last_print = init_venv_check_start 441 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 442 if not lock_path.exists(): 443 break 444 445 now = time.perf_counter() 446 if debug or (now - last_print) > sleep_message_seconds: 447 print(f"Lock exists for venv '{venv}', sleeping...") 448 last_print = now 449 time.sleep(step_sleep_seconds) 450 update_lock(False) 451 452 if ( 453 not force 454 and venv == 'mrsm' 455 and os.environ.get(work_dir_env_var, None) is not None 456 and docker_home_venv_path.exists() 457 ): 458 wait_for_lock() 459 shutil.move(docker_home_venv_path, venv_path) 460 if verify: 461 verify_venv(venv, debug=debug) 462 verified_venvs.add(venv) 463 return True 464 465 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 466 global tried_virtualenv 467 try: 468 import venv as _venv 469 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 470 virtualenv = None 471 except ImportError: 472 _venv = None 473 uv = None 474 virtualenv = None 475 476 _venv_success = False 477 temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv) 478 ### NOTE: Disable site-packages movement for now. 479 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 480 481 wait_for_lock() 482 update_lock(True) 483 484 if rename_vtp: 485 if debug: 486 print(f"Moving '{vtp}' to '{temp_vtp}'...") 487 shutil.move(vtp, temp_vtp) 488 489 if uv is not None: 490 _venv_success = run_python_package( 491 'uv', 492 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 493 venv=None, 494 env=_get_pip_os_env(), 495 debug=debug, 496 ) == 0 497 498 if _venv is not None and not _venv_success: 499 f = io.StringIO() 500 with redirect_stdout(f): 501 _venv_success = run_python_package( 502 'venv', 503 [venv_path.as_posix()] + ( 504 ['--symlinks'] 505 if platform.system() != 'Windows' 506 else [] 507 ), 508 venv=None, debug=debug 509 ) == 0 510 if not _venv_success: 511 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 512 if not venv_exists(venv, debug=debug): 513 _venv = None 514 if not _venv_success: 515 virtualenv = attempt_import( 516 'virtualenv', 517 venv=None, 518 lazy=False, 519 install=(not tried_virtualenv), 520 warn=False, 521 check_update=False, 522 color=False, 523 debug=debug, 524 ) 525 if virtualenv is None: 526 print( 527 "Failed to import `venv` or `virtualenv`! " 528 + "Please install `virtualenv` via pip then restart Meerschaum." 529 ) 530 if rename_vtp and temp_vtp.exists(): 531 if debug: 532 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 533 shutil.move(temp_vtp, vtp) 534 update_lock(False) 535 return False 536 537 tried_virtualenv = True 538 try: 539 python_folder = ( 540 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 541 ) 542 dist_packages_path = ( 543 VIRTENV_RESOURCES_PATH / 544 venv / 'local' / 'lib' / python_folder / 'dist-packages' 545 ) 546 local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 547 bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin' 548 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 549 if bin_path.exists(): 550 try: 551 shutil.rmtree(bin_path) 552 except Exception: 553 import traceback 554 traceback.print_exc() 555 virtualenv.cli_run([venv_path.as_posix()]) 556 if dist_packages_path.exists(): 557 vtp.mkdir(exist_ok=True, parents=True) 558 for file_path in dist_packages_path.glob('*'): 559 shutil.move(file_path, vtp) 560 shutil.rmtree(dist_packages_path) 561 # shutil.move(dist_packages_path, vtp) 562 bin_path.mkdir(exist_ok=True, parents=True) 563 for file_path in local_bin_path.glob('*'): 564 shutil.move(file_path, bin_path) 565 # shutil.move(local_bin_path, bin_path) 566 shutil.rmtree(local_bin_path) 567 568 except Exception: 569 import traceback 570 traceback.print_exc() 571 if rename_vtp and temp_vtp.exists(): 572 shutil.move(temp_vtp, vtp) 573 update_lock(False) 574 return False 575 if verify: 576 verify_venv(venv, debug=debug) 577 verified_venvs.add(venv) 578 579 if rename_vtp and temp_vtp.exists(): 580 if debug: 581 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 582 shutil.move(temp_vtp, vtp) 583 584 update_lock(False) 585 return True
Initialize the virtual environment.
Parameters
- venv (str, default 'mrsm'): The name of the virtual environment to create.
- verify (bool, default True):
If
True
, verify that the virtual environment is in the expected state. - force (bool, default False):
If
True
, recreate the virtual environment, even if already initalized.
Returns
- A
bool
indicating success.
820def inside_venv() -> bool: 821 """ 822 Determine whether current Python interpreter is running inside a virtual environment. 823 """ 824 import sys 825 return ( 826 hasattr(sys, 'real_prefix') or ( 827 hasattr(sys, 'base_prefix') 828 and sys.base_prefix != sys.prefix 829 ) 830 )
Determine whether current Python interpreter is running inside a virtual environment.
212def is_venv_active( 213 venv: str = 'mrsm', 214 color : bool = True, 215 debug: bool = False 216) -> bool: 217 """ 218 Check if a virtual environment is active. 219 220 Parameters 221 ---------- 222 venv: str, default 'mrsm' 223 The virtual environment to check. 224 225 color: bool, default True 226 If `True`, include color in debug text. 227 228 debug: bool, default False 229 Verbosity toggle. 230 231 Returns 232 ------- 233 A bool indicating whether the virtual environment `venv` is active. 234 235 """ 236 return venv in active_venvs
Check if a virtual environment is active.
Parameters
- venv (str, default 'mrsm'): The virtual environment to check.
- color (bool, default True):
If
True
, include color in debug text. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment
venv
is active.
613def venv_exec( 614 code: str, 615 venv: Optional[str] = 'mrsm', 616 env: Optional[Dict[str, str]] = None, 617 with_extras: bool = False, 618 as_proc: bool = False, 619 capture_output: bool = True, 620 debug: bool = False, 621) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 622 """ 623 Execute Python code in a subprocess via a virtual environment's interpeter. 624 Return `True` if the code successfully executes, `False` on failure. 625 626 Parameters 627 ---------- 628 code: str 629 The Python code to excecute. 630 631 venv: str, default 'mrsm' 632 The virtual environment to use to get the path for the Python executable. 633 If `venv` is `None`, use the default `sys.executable` path. 634 635 env: Optional[Dict[str, str]], default None 636 Optionally specify the environment variables for the subprocess. 637 Defaults to `os.environ`. 638 639 with_extras: bool, default False 640 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 641 642 as_proc: bool, default False 643 If `True`, return the `subprocess.Popen` object instead of executing. 644 645 Returns 646 ------- 647 By default, return a bool indicating success. 648 If `as_proc` is `True`, return a `subprocess.Popen` object. 649 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 650 651 """ 652 import os 653 import subprocess 654 import platform 655 from meerschaum.utils.debug import dprint 656 from meerschaum.utils.process import _child_processes 657 658 executable = venv_executable(venv=venv) 659 cmd_list = [executable, '-c', code] 660 if env is None: 661 env = os.environ 662 if debug: 663 dprint(str(cmd_list)) 664 if not with_extras and not as_proc: 665 return subprocess.call(cmd_list, env=env) == 0 666 667 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 668 group_kwargs = ( 669 { 670 'preexec_fn': os.setsid, 671 } if platform.system() != 'Windows' 672 else { 673 'creationflags': CREATE_NEW_PROCESS_GROUP, 674 } 675 ) 676 process = subprocess.Popen( 677 cmd_list, 678 stdout=stdout, 679 stderr=stderr, 680 stdin=sys.stdin, 681 env=env, 682 **group_kwargs 683 ) 684 if as_proc: 685 _child_processes.append(process) 686 return process 687 stdout, stderr = process.communicate() 688 exit_code = process.returncode 689 return exit_code, stdout, stderr
Execute Python code in a subprocess via a virtual environment's interpeter.
Return True
if the code successfully executes, False
on failure.
Parameters
- code (str): The Python code to excecute.
- venv (str, default 'mrsm'):
The virtual environment to use to get the path for the Python executable.
If
venv
isNone
, use the defaultsys.executable
path. - env (Optional[Dict[str, str]], default None):
Optionally specify the environment variables for the subprocess.
Defaults to
os.environ
. - with_extras (bool, default False):
If
True
, return a tuple of the exit code, stdout bytes, and stderr bytes. - as_proc (bool, default False):
If
True
, return thesubprocess.Popen
object instead of executing.
Returns
- By default, return a bool indicating success.
- If
as_proc
isTrue
, return asubprocess.Popen
object. - If
with_extras
isTrue
, return a tuple of the exit code, stdout bytes, and stderr bytes.
588def venv_executable(venv: Optional[str] = 'mrsm') -> str: 589 """ 590 The Python interpreter executable for a given virtual environment. 591 """ 592 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 593 import sys 594 import platform 595 return ( 596 sys.executable if venv is None 597 else str( 598 VIRTENV_RESOURCES_PATH 599 / venv 600 / ( 601 'bin' if platform.system() != 'Windows' 602 else 'Scripts' 603 ) / ( 604 'python' 605 + str(sys.version_info.major) 606 + '.' 607 + str(sys.version_info.minor) 608 ) 609 ) 610 )
The Python interpreter executable for a given virtual environment.
692def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 693 """ 694 Determine whether a virtual environment has been created. 695 """ 696 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 697 return target_path.exists()
Determine whether a virtual environment has been created.
700def venv_target_path( 701 venv: Union[str, None], 702 allow_nonexistent: bool = False, 703 debug: bool = False, 704) -> pathlib.Path: 705 """ 706 Return a virtual environment's site-package path. 707 708 Parameters 709 ---------- 710 venv: Union[str, None] 711 The virtual environment for which a path should be returned. 712 713 allow_nonexistent: bool, default False 714 If `True`, return a path even if it does not exist. 715 716 Returns 717 ------- 718 The `pathlib.Path` object for the virtual environment's path. 719 720 """ 721 import os 722 import sys 723 import platform 724 import site 725 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 726 from meerschaum._internal.static import STATIC_CONFIG 727 728 ### Check sys.path for a user-writable site-packages directory. 729 if venv is None: 730 731 ### Return the known value for the portable environment. 732 environment_runtime = STATIC_CONFIG['environment']['runtime'] 733 if os.environ.get(environment_runtime, None) == 'portable': 734 python_version_folder = ( 735 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 736 ) 737 executable_path = pathlib.Path(sys.executable) 738 site_packages_path = ( 739 ( 740 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 741 ) if platform.system() != 'Windows' else ( 742 executable_path.parent / 'Lib' / 'site-packages' 743 ) 744 ) 745 if not site_packages_path.exists(): 746 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 747 return site_packages_path 748 749 if not inside_venv(): 750 user_site_packages = site.getusersitepackages() 751 if user_site_packages is None: 752 raise EnvironmentError("Could not determine user site packages.") 753 754 site_path = pathlib.Path(user_site_packages) 755 if not site_path.exists(): 756 757 ### Windows does not have `os.geteuid()`. 758 if platform.system() == 'Windows' or os.geteuid() != 0: 759 site_path.mkdir(parents=True, exist_ok=True) 760 return site_path 761 762 ### Allow for dist-level paths (running as root). 763 for possible_dist in site.getsitepackages(): 764 dist_path = pathlib.Path(possible_dist) 765 if not dist_path.exists(): 766 continue 767 return dist_path 768 769 raise EnvironmentError("Could not determine the dist-packages directory.") 770 771 return site_path 772 773 venv_root_path = ( 774 (VIRTENV_RESOURCES_PATH / venv) 775 if venv is not None 776 else pathlib.Path(sys.prefix) 777 ) 778 target_path = venv_root_path 779 780 ### Ensure 'lib' or 'Lib' exists. 781 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 782 if not allow_nonexistent: 783 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 784 print(f"Failed to find lib directory for virtual environment '{venv}'.") 785 import traceback 786 traceback.print_stack() 787 sys.exit(1) 788 target_path = target_path / lib 789 790 ### Check if a 'python3.x' folder exists. 791 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 792 if target_path.exists(): 793 target_path = ( 794 (target_path / python_folder) 795 if python_folder in os.listdir(target_path) 796 else target_path 797 ) 798 else: 799 target_path = ( 800 (target_path / python_folder) 801 if platform.system() != 'Windows' 802 else target_path 803 ) 804 805 ### Ensure 'site-packages' exists. 806 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 807 target_path = target_path / 'site-packages' 808 else: 809 import traceback 810 traceback.print_stack() 811 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 812 print("This may be because you are using a different Python version.") 813 print("Try deleting the following directory and restarting Meerschaum:") 814 print(VIRTENV_RESOURCES_PATH) 815 sys.exit(1) 816 817 return target_path
Return a virtual environment's site-package path.
Parameters
- venv (Union[str, None]): The virtual environment for which a path should be returned.
- allow_nonexistent (bool, default False):
If
True
, return a path even if it does not exist.
Returns
- The
pathlib.Path
object for the virtual environment's path.
240def verify_venv( 241 venv: str, 242 debug: bool = False, 243) -> None: 244 """ 245 Verify that the virtual environment matches the expected state. 246 """ 247 import platform 248 import os 249 import shutil 250 import sys 251 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 252 from meerschaum.utils.process import run_process 253 from meerschaum.utils.misc import make_symlink, is_symlink 254 from meerschaum.utils.warnings import warn 255 256 venv_path = VIRTENV_RESOURCES_PATH / venv 257 bin_path = venv_path / ( 258 'bin' if platform.system() != 'Windows' else "Scripts" 259 ) 260 current_python_versioned_name = ( 261 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 262 + ('' if platform.system() != 'Windows' else '.exe') 263 ) 264 265 if not (bin_path / current_python_versioned_name).exists(): 266 init_venv(venv, verify=False, force=True, debug=debug) 267 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 268 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 269 if not current_python_in_venv_path.exists(): 270 if is_symlink(current_python_in_venv_path): 271 try: 272 current_python_in_venv_path.unlink() 273 except Exception as e: 274 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 275 try: 276 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 277 except Exception: 278 print( 279 f"Unable to create symlink {current_python_in_venv_path} " 280 + f"to {current_python_in_sys_path}." 281 ) 282 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 283 else: 284 files_to_inspect = [current_python_versioned_name] 285 286 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 287 """ 288 Return the version for the python binary at the given path. 289 """ 290 try: 291 ### It might be a broken symlink, so skip on errors. 292 if debug: 293 print(f"Getting python version for {python_path}") 294 proc = run_process( 295 [str(python_path), '-V'], 296 as_proc=True, 297 capture_output=True, 298 ) 299 stdout, stderr = proc.communicate(timeout=1.0) 300 except Exception as e: 301 ### E.g. the symlink may be broken. 302 if is_symlink(python_path): 303 try: 304 python_path.unlink() 305 except Exception as _e: 306 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 307 return None 308 return stdout.decode('utf-8').strip().replace('Python ', '') 309 310 ### Ensure the versions are symlinked correctly. 311 for filename in files_to_inspect: 312 if not filename.startswith('python'): 313 continue 314 python_path = bin_path / filename 315 version = get_python_version(python_path) 316 if version is None: 317 continue 318 try: 319 major_version = version.split('.', maxsplit=1)[0] 320 minor_version = version.split('.', maxsplit=2)[1] 321 except IndexError: 322 return 323 python_versioned_name = ( 324 'python' + major_version + '.' + minor_version 325 + ('' if platform.system() != 'Windows' else '.exe') 326 ) 327 328 ### E.g. python3.10 actually links to Python 3.10. 329 if filename == python_versioned_name: 330 try: 331 real_path = pathlib.Path(os.path.realpath(python_path)) 332 real_path_exists = real_path.exists() 333 except Exception: 334 real_path_exists = False 335 336 if not real_path_exists: 337 try: 338 python_path.unlink() 339 except Exception: 340 pass 341 init_venv(venv, verify=False, force=True, debug=debug) 342 if not python_path.exists(): 343 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 344 345 if python_path == real_path: 346 continue 347 348 try: 349 python_path.unlink() 350 except Exception: 351 pass 352 success, msg = make_symlink(real_path, python_path) 353 if not success: 354 warn(msg, color=False) 355 continue 356 357 python_versioned_path = bin_path / python_versioned_name 358 if python_versioned_path.exists(): 359 ### Avoid circular symlinks. 360 if get_python_version(python_versioned_path) == version: 361 continue 362 python_versioned_path.unlink() 363 shutil.move(python_path, python_versioned_path)
Verify that the virtual environment matches the expected state.