meerschaum.utils.venv
Manage virtual environments.
1#! /usr/bin/env python3 2# -*- coding: utf-8 -*- 3# vim:fenc=utf-8 4 5""" 6Manage virtual environments. 7""" 8 9from __future__ import annotations 10 11import sys 12import pathlib 13 14from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple 15from meerschaum.utils.threading import RLock, get_ident 16 17__all__ = sorted([ 18 'activate_venv', 'deactivate_venv', 'init_venv', 19 'inside_venv', 'is_venv_active', 'venv_exec', 20 'venv_executable', 'venv_exists', 'venv_target_path', 21 'Venv', 'get_venvs', 'verify_venv', 'get_module_venv', 22]) 23__pdoc__ = {'Venv': True} 24 25LOCKS = { 26 'sys.path': RLock(), 27 'active_venvs': RLock(), 28 'venvs_active_counts': RLock(), 29} 30 31active_venvs = set() 32active_venvs_counts: Dict[str, int] = {} 33active_venvs_order: List[Optional[str]] = [] 34threads_active_venvs: Dict[int, 'set[str]'] = {} 35CREATE_NEW_PROCESS_GROUP = 0x00000200 36 37 38def activate_venv( 39 venv: Optional[str] = 'mrsm', 40 color: bool = True, 41 force: bool = False, 42 init_if_not_exists: bool = True, 43 debug: bool = False, 44 **kw 45) -> bool: 46 """ 47 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 48 49 Parameters 50 ---------- 51 venv: Optional[str], default 'mrsm' 52 The virtual environment to activate. 53 54 color: bool, default True 55 If `True`, include color in debug text. 56 57 init_if_not_exists: bool, default True 58 If `True`, create the virtual environment if it does not exist. 59 60 force: bool, default False 61 If `True`, do not exit early even if the venv is currently active. 62 63 debug: bool, default False 64 Verbosity toggle. 65 66 Returns 67 ------- 68 A bool indicating whether the virtual environment was successfully activated. 69 70 """ 71 thread_id = get_ident() 72 if active_venvs_order and active_venvs_order[0] == venv: 73 if not force: 74 return True 75 76 import sys 77 import os 78 if venv is not None and init_if_not_exists: 79 init_venv(venv=venv, debug=debug) 80 81 with LOCKS['active_venvs']: 82 if thread_id not in threads_active_venvs: 83 threads_active_venvs[thread_id] = {} 84 active_venvs.add(venv) 85 if venv not in threads_active_venvs[thread_id]: 86 threads_active_venvs[thread_id][venv] = 1 87 else: 88 threads_active_venvs[thread_id][venv] += 1 89 90 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 91 if not target_path.exists() and venv is not None and init_if_not_exists: 92 init_venv(venv=venv, force=True, debug=debug) 93 94 if not target_path.exists(): 95 if init_if_not_exists: 96 return False 97 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 98 99 target = target_path.as_posix() 100 101 if venv in active_venvs_order: 102 try: 103 sys.path.remove(target) 104 except Exception: 105 pass 106 try: 107 active_venvs_order.remove(venv) 108 except Exception: 109 pass 110 111 if venv is not None: 112 sys.path.insert(0, target) 113 else: 114 if sys.path and sys.path[0] in (os.getcwd(), ''): 115 sys.path.insert(1, target) 116 else: 117 sys.path.insert(0, target) 118 try: 119 active_venvs_order.insert(0, venv) 120 except Exception: 121 pass 122 123 return True 124 125 126def deactivate_venv( 127 venv: str = 'mrsm', 128 color: bool = True, 129 debug: bool = False, 130 previously_active_venvs: Union['set[str]', List[str], None] = None, 131 force: bool = False, 132 **kw 133) -> bool: 134 """ 135 Remove a virtual environment from `sys.path` (if it's been activated). 136 137 Parameters 138 ---------- 139 venv: str, default 'mrsm' 140 The virtual environment to deactivate. 141 142 color: bool, default True 143 If `True`, include color in debug text. 144 145 debug: bool, default False 146 Verbosity toggle. 147 148 previously_active_venvs: Union[Set[str], List[str], None] 149 If provided, skip deactivating if a virtual environment is in this iterable. 150 151 force: bool, default False 152 If `True`, forcibly deactivate the virtual environment. 153 This may cause issues with other threads, so be careful! 154 155 Returns 156 ------- 157 Return a bool indicating whether the virtual environment was successfully deactivated. 158 159 """ 160 import sys 161 thread_id = get_ident() 162 if venv is None: 163 if venv in active_venvs: 164 active_venvs.remove(venv) 165 return True 166 167 if previously_active_venvs and venv in previously_active_venvs and not force: 168 return True 169 170 with LOCKS['active_venvs']: 171 if venv in threads_active_venvs.get(thread_id, {}): 172 new_count = threads_active_venvs[thread_id][venv] - 1 173 if new_count > 0 and not force: 174 threads_active_venvs[thread_id][venv] = new_count 175 return True 176 else: 177 del threads_active_venvs[thread_id][venv] 178 179 if not force: 180 for other_thread_id, other_venvs in threads_active_venvs.items(): 181 if other_thread_id == thread_id: 182 continue 183 if venv in other_venvs: 184 return True 185 else: 186 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 187 for other_thread_id in to_delete: 188 del threads_active_venvs[other_thread_id] 189 190 if venv in active_venvs: 191 active_venvs.remove(venv) 192 193 if sys.path is None: 194 return False 195 196 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 197 with LOCKS['sys.path']: 198 if target in sys.path: 199 try: 200 sys.path.remove(target) 201 except Exception: 202 pass 203 try: 204 active_venvs_order.remove(venv) 205 except Exception: 206 pass 207 208 return True 209 210 211def is_venv_active( 212 venv: str = 'mrsm', 213 color : bool = True, 214 debug: bool = False 215) -> bool: 216 """ 217 Check if a virtual environment is active. 218 219 Parameters 220 ---------- 221 venv: str, default 'mrsm' 222 The virtual environment to check. 223 224 color: bool, default True 225 If `True`, include color in debug text. 226 227 debug: bool, default False 228 Verbosity toggle. 229 230 Returns 231 ------- 232 A bool indicating whether the virtual environment `venv` is active. 233 234 """ 235 return venv in active_venvs 236 237 238verified_venvs = set() 239def verify_venv( 240 venv: str, 241 debug: bool = False, 242) -> None: 243 """ 244 Verify that the virtual environment matches the expected state. 245 """ 246 import platform 247 import os 248 import shutil 249 import sys 250 251 import meerschaum.config.paths as paths 252 from meerschaum.utils.process import run_process 253 from meerschaum.utils.misc import make_symlink, is_symlink 254 from meerschaum.utils.warnings import warn 255 256 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 257 bin_path = venv_path / ( 258 'bin' if platform.system() != 'Windows' else "Scripts" 259 ) 260 current_python_versioned_name = ( 261 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 262 + ('' if platform.system() != 'Windows' else '.exe') 263 ) 264 265 if not (bin_path / current_python_versioned_name).exists(): 266 init_venv(venv, verify=False, force=True, debug=debug) 267 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 268 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 269 if not current_python_in_venv_path.exists(): 270 if is_symlink(current_python_in_venv_path): 271 try: 272 current_python_in_venv_path.unlink() 273 except Exception as e: 274 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 275 try: 276 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 277 except Exception: 278 print( 279 f"Unable to create symlink {current_python_in_venv_path} " 280 + f"to {current_python_in_sys_path}." 281 ) 282 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 283 else: 284 files_to_inspect = [current_python_versioned_name] 285 286 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 287 """ 288 Return the version for the python binary at the given path. 289 """ 290 try: 291 ### It might be a broken symlink, so skip on errors. 292 if debug: 293 print(f"Getting python version for {python_path}") 294 proc = run_process( 295 [str(python_path), '-V'], 296 as_proc=True, 297 capture_output=True, 298 ) 299 stdout, stderr = proc.communicate(timeout=1.0) 300 except Exception as e: 301 ### E.g. the symlink may be broken. 302 if is_symlink(python_path): 303 try: 304 python_path.unlink() 305 except Exception as _e: 306 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 307 return None 308 return stdout.decode('utf-8').strip().replace('Python ', '') 309 310 ### Ensure the versions are symlinked correctly. 311 for filename in files_to_inspect: 312 if not filename.startswith('python'): 313 continue 314 python_path = bin_path / filename 315 version = get_python_version(python_path) 316 if version is None: 317 continue 318 try: 319 major_version = version.split('.', maxsplit=1)[0] 320 minor_version = version.split('.', maxsplit=2)[1] 321 except IndexError: 322 return 323 python_versioned_name = ( 324 'python' + major_version + '.' + minor_version 325 + ('' if platform.system() != 'Windows' else '.exe') 326 ) 327 328 ### E.g. python3.10 actually links to Python 3.10. 329 if filename == python_versioned_name: 330 try: 331 real_path = pathlib.Path(os.path.realpath(python_path)) 332 real_path_exists = real_path.exists() 333 except Exception: 334 real_path_exists = False 335 336 if not real_path_exists: 337 try: 338 python_path.unlink() 339 except Exception: 340 pass 341 init_venv(venv, verify=False, force=True, debug=debug) 342 if not python_path.exists(): 343 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 344 345 if python_path == real_path: 346 continue 347 348 try: 349 python_path.unlink() 350 except Exception: 351 pass 352 success, msg = make_symlink(real_path, python_path) 353 if not success: 354 warn(msg, color=False) 355 continue 356 357 python_versioned_path = bin_path / python_versioned_name 358 if python_versioned_path.exists(): 359 ### Avoid circular symlinks. 360 if get_python_version(python_versioned_path) == version: 361 continue 362 python_versioned_path.unlink() 363 shutil.move(python_path, python_versioned_path) 364 365 366tried_virtualenv = False 367def init_venv( 368 venv: str = 'mrsm', 369 verify: bool = True, 370 force: bool = False, 371 debug: bool = False, 372) -> bool: 373 """ 374 Initialize the virtual environment. 375 376 Parameters 377 ---------- 378 venv: str, default 'mrsm' 379 The name of the virtual environment to create. 380 381 verify: bool, default True 382 If `True`, verify that the virtual environment is in the expected state. 383 384 force: bool, default False 385 If `True`, recreate the virtual environment, even if already initalized. 386 387 Returns 388 ------- 389 A `bool` indicating success. 390 """ 391 if not force and venv in verified_venvs: 392 return True 393 if not force and venv_exists(venv, debug=debug): 394 if verify: 395 verify_venv(venv, debug=debug) 396 verified_venvs.add(venv) 397 return True 398 399 import io 400 from contextlib import redirect_stdout 401 import sys 402 import platform 403 import os 404 import shutil 405 import time 406 407 import meerschaum.config.paths as paths 408 from meerschaum._internal.static import STATIC_CONFIG 409 from meerschaum.utils.packages import is_uv_enabled 410 411 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 412 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 413 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 414 lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 415 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 416 417 def update_lock(active: bool): 418 try: 419 if not active: 420 if debug: 421 print(f"Releasing lock: '{lock_path}'") 422 lock_path.unlink() 423 else: 424 if debug: 425 print(f"Acquiring lock: '{lock_path}'") 426 lock_path.touch() 427 except Exception: 428 pass 429 430 def wait_for_lock(): 431 if platform.system() == 'Windows': 432 return 433 max_lock_seconds = 30.0 434 sleep_message_seconds = 5.0 435 step_sleep_seconds = 0.1 436 init_venv_check_start = time.perf_counter() 437 last_print = init_venv_check_start 438 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 439 if not lock_path.exists(): 440 break 441 442 now = time.perf_counter() 443 if debug or (now - last_print) > sleep_message_seconds: 444 print(f"Lock exists for venv '{venv}', sleeping...") 445 last_print = now 446 time.sleep(step_sleep_seconds) 447 update_lock(False) 448 449 if ( 450 not force 451 and venv == 'mrsm' 452 and os.environ.get(work_dir_env_var, None) is not None 453 and docker_home_venv_path.exists() 454 ): 455 wait_for_lock() 456 shutil.move(docker_home_venv_path, venv_path) 457 if verify: 458 verify_venv(venv, debug=debug) 459 verified_venvs.add(venv) 460 return True 461 462 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 463 global tried_virtualenv 464 try: 465 import venv as _venv 466 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 467 virtualenv = None 468 except ImportError: 469 _venv = None 470 uv = None 471 virtualenv = None 472 473 _venv_success = False 474 temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv) 475 ### NOTE: Disable site-packages movement for now. 476 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 477 478 wait_for_lock() 479 update_lock(True) 480 481 if rename_vtp: 482 if debug: 483 print(f"Moving '{vtp}' to '{temp_vtp}'...") 484 shutil.move(vtp, temp_vtp) 485 486 if uv is not None: 487 _venv_success = run_python_package( 488 'uv', 489 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 490 venv=None, 491 env=_get_pip_os_env(), 492 debug=debug, 493 ) == 0 494 495 if _venv is not None and not _venv_success: 496 f = io.StringIO() 497 with redirect_stdout(f): 498 _venv_success = run_python_package( 499 'venv', 500 [venv_path.as_posix()] + ( 501 ['--symlinks'] 502 if platform.system() != 'Windows' 503 else [] 504 ), 505 venv=None, debug=debug 506 ) == 0 507 if not _venv_success: 508 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 509 if not venv_exists(venv, debug=debug): 510 _venv = None 511 if not _venv_success: 512 virtualenv = attempt_import( 513 'virtualenv', 514 venv=None, 515 lazy=False, 516 install=(not tried_virtualenv), 517 warn=False, 518 check_update=False, 519 color=False, 520 debug=debug, 521 ) 522 if virtualenv is None: 523 print( 524 "Failed to import `venv` or `virtualenv`! " 525 + "Please install `virtualenv` via pip then restart Meerschaum." 526 ) 527 if rename_vtp and temp_vtp.exists(): 528 if debug: 529 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 530 shutil.move(temp_vtp, vtp) 531 update_lock(False) 532 return False 533 534 tried_virtualenv = True 535 try: 536 python_folder = ( 537 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 538 ) 539 dist_packages_path = ( 540 paths.VIRTENV_RESOURCES_PATH / 541 venv / 'local' / 'lib' / python_folder / 'dist-packages' 542 ) 543 local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 544 bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin' 545 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 546 if bin_path.exists(): 547 try: 548 shutil.rmtree(bin_path) 549 except Exception: 550 import traceback 551 traceback.print_exc() 552 virtualenv.cli_run([venv_path.as_posix()]) 553 if dist_packages_path.exists(): 554 vtp.mkdir(exist_ok=True, parents=True) 555 for file_path in dist_packages_path.glob('*'): 556 shutil.move(file_path, vtp) 557 shutil.rmtree(dist_packages_path) 558 # shutil.move(dist_packages_path, vtp) 559 bin_path.mkdir(exist_ok=True, parents=True) 560 for file_path in local_bin_path.glob('*'): 561 shutil.move(file_path, bin_path) 562 # shutil.move(local_bin_path, bin_path) 563 shutil.rmtree(local_bin_path) 564 565 except Exception: 566 import traceback 567 traceback.print_exc() 568 if rename_vtp and temp_vtp.exists(): 569 shutil.move(temp_vtp, vtp) 570 update_lock(False) 571 return False 572 if verify: 573 verify_venv(venv, debug=debug) 574 verified_venvs.add(venv) 575 576 if rename_vtp and temp_vtp.exists(): 577 if debug: 578 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 579 shutil.move(temp_vtp, vtp) 580 581 update_lock(False) 582 return True 583 584 585def venv_executable(venv: Optional[str] = 'mrsm') -> str: 586 """ 587 The Python interpreter executable for a given virtual environment. 588 """ 589 import sys 590 import platform 591 import meerschaum.config.paths as paths 592 return ( 593 sys.executable if venv is None 594 else str( 595 paths.VIRTENV_RESOURCES_PATH 596 / venv 597 / ( 598 'bin' if platform.system() != 'Windows' 599 else 'Scripts' 600 ) / ( 601 'python' 602 + str(sys.version_info.major) 603 + '.' 604 + str(sys.version_info.minor) 605 ) 606 ) 607 ) 608 609 610def venv_exec( 611 code: str, 612 venv: Optional[str] = 'mrsm', 613 env: Optional[Dict[str, str]] = None, 614 with_extras: bool = False, 615 as_proc: bool = False, 616 capture_output: bool = True, 617 debug: bool = False, 618) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 619 """ 620 Execute Python code in a subprocess via a virtual environment's interpeter. 621 Return `True` if the code successfully executes, `False` on failure. 622 623 Parameters 624 ---------- 625 code: str 626 The Python code to excecute. 627 628 venv: str, default 'mrsm' 629 The virtual environment to use to get the path for the Python executable. 630 If `venv` is `None`, use the default `sys.executable` path. 631 632 env: Optional[Dict[str, str]], default None 633 Optionally specify the environment variables for the subprocess. 634 Defaults to `os.environ`. 635 636 with_extras: bool, default False 637 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 638 639 as_proc: bool, default False 640 If `True`, return the `subprocess.Popen` object instead of executing. 641 642 Returns 643 ------- 644 By default, return a bool indicating success. 645 If `as_proc` is `True`, return a `subprocess.Popen` object. 646 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 647 648 """ 649 import os 650 import subprocess 651 import platform 652 from meerschaum.utils.debug import dprint 653 from meerschaum.utils.process import _child_processes 654 655 executable = venv_executable(venv=venv) 656 cmd_list = [executable, '-c', code] 657 if env is None: 658 env = os.environ 659 if debug: 660 dprint(str(cmd_list)) 661 if not with_extras and not as_proc: 662 return subprocess.call(cmd_list, env=env) == 0 663 664 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 665 group_kwargs = ( 666 { 667 'preexec_fn': os.setsid, 668 } if platform.system() != 'Windows' 669 else { 670 'creationflags': CREATE_NEW_PROCESS_GROUP, 671 } 672 ) 673 process = subprocess.Popen( 674 cmd_list, 675 stdout=stdout, 676 stderr=stderr, 677 stdin=sys.stdin, 678 env=env, 679 **group_kwargs 680 ) 681 if as_proc: 682 _child_processes.append(process) 683 return process 684 stdout, stderr = process.communicate() 685 exit_code = process.returncode 686 return exit_code, stdout, stderr 687 688 689def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 690 """ 691 Determine whether a virtual environment has been created. 692 """ 693 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 694 return target_path.exists() 695 696 697def venv_target_path( 698 venv: Union[str, None], 699 allow_nonexistent: bool = False, 700 debug: bool = False, 701) -> pathlib.Path: 702 """ 703 Return a virtual environment's site-package path. 704 705 Parameters 706 ---------- 707 venv: Union[str, None] 708 The virtual environment for which a path should be returned. 709 710 allow_nonexistent: bool, default False 711 If `True`, return a path even if it does not exist. 712 713 Returns 714 ------- 715 The `pathlib.Path` object for the virtual environment's path. 716 717 """ 718 import os 719 import sys 720 import platform 721 import site 722 723 import meerschaum.config.paths as paths 724 from meerschaum._internal.static import STATIC_CONFIG 725 726 ### Check sys.path for a user-writable site-packages directory. 727 if venv is None: 728 729 ### Return the known value for the portable environment. 730 environment_runtime = STATIC_CONFIG['environment']['runtime'] 731 if os.environ.get(environment_runtime, None) == 'portable': 732 python_version_folder = ( 733 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 734 ) 735 executable_path = pathlib.Path(sys.executable) 736 site_packages_path = ( 737 ( 738 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 739 ) if platform.system() != 'Windows' else ( 740 executable_path.parent / 'Lib' / 'site-packages' 741 ) 742 ) 743 if not site_packages_path.exists(): 744 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 745 return site_packages_path 746 747 if not inside_venv(): 748 user_site_packages = site.getusersitepackages() 749 if user_site_packages is None: 750 raise EnvironmentError("Could not determine user site packages.") 751 752 site_path = pathlib.Path(user_site_packages) 753 if not site_path.exists(): 754 755 ### Windows does not have `os.geteuid()`. 756 if platform.system() == 'Windows' or os.geteuid() != 0: 757 site_path.mkdir(parents=True, exist_ok=True) 758 return site_path 759 760 ### Allow for dist-level paths (running as root). 761 for possible_dist in site.getsitepackages(): 762 dist_path = pathlib.Path(possible_dist) 763 if not dist_path.exists(): 764 continue 765 return dist_path 766 767 raise EnvironmentError("Could not determine the dist-packages directory.") 768 769 return site_path 770 771 venv_root_path = ( 772 (paths.VIRTENV_RESOURCES_PATH / venv) 773 if venv is not None 774 else pathlib.Path(sys.prefix) 775 ) 776 target_path = venv_root_path 777 778 ### Ensure 'lib' or 'Lib' exists. 779 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 780 if not allow_nonexistent: 781 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 782 print(f"Failed to find lib directory for virtual environment '{venv}'.") 783 import traceback 784 traceback.print_stack() 785 sys.exit(1) 786 target_path = target_path / lib 787 788 ### Check if a 'python3.x' folder exists. 789 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 790 if target_path.exists(): 791 target_path = ( 792 (target_path / python_folder) 793 if python_folder in os.listdir(target_path) 794 else target_path 795 ) 796 else: 797 target_path = ( 798 (target_path / python_folder) 799 if platform.system() != 'Windows' 800 else target_path 801 ) 802 803 ### Ensure 'site-packages' exists. 804 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 805 target_path = target_path / 'site-packages' 806 else: 807 import traceback 808 traceback.print_stack() 809 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 810 print("This may be because you are using a different Python version.") 811 print("Try deleting the following directory and restarting Meerschaum:") 812 print(paths.VIRTENV_RESOURCES_PATH) 813 sys.exit(1) 814 815 return target_path 816 817 818def inside_venv() -> bool: 819 """ 820 Determine whether current Python interpreter is running inside a virtual environment. 821 """ 822 import sys 823 return ( 824 hasattr(sys, 'real_prefix') or ( 825 hasattr(sys, 'base_prefix') 826 and sys.base_prefix != sys.prefix 827 ) 828 ) 829 830 831def get_venvs() -> List[str]: 832 """ 833 Return a list of all the virtual environments. 834 """ 835 import os 836 import meerschaum.config.paths as paths 837 venvs = [] 838 for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH): 839 path = paths.VIRTENV_RESOURCES_PATH / filename 840 if not path.is_dir(): 841 continue 842 if not venv_exists(filename): 843 continue 844 venvs.append(filename) 845 return venvs 846 847 848def get_module_venv(module) -> Union[str, None]: 849 """ 850 Return the virtual environment where an imported module is installed. 851 852 Parameters 853 ---------- 854 module: ModuleType 855 The imported module to inspect. 856 857 Returns 858 ------- 859 The name of a venv or `None`. 860 """ 861 import meerschaum.config.paths as paths 862 module_path = pathlib.Path(module.__file__).resolve() 863 try: 864 rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH) 865 except ValueError: 866 return None 867 868 return rel_path.as_posix().split('/', maxsplit=1)[0] 869 870 871from meerschaum.utils.venv._Venv import Venv
19class Venv: 20 """ 21 Manage a virtual enviroment's activation status. 22 23 Examples 24 -------- 25 >>> from meerschaum.plugins import Plugin 26 >>> with Venv('mrsm') as venv: 27 ... import pandas 28 >>> with Venv(Plugin('noaa')) as venv: 29 ... import requests 30 >>> venv = Venv('mrsm') 31 >>> venv.activate() 32 True 33 >>> venv.deactivate() 34 True 35 >>> 36 """ 37 38 def __init__( 39 self, 40 venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm', 41 init_if_not_exists: bool = True, 42 debug: bool = False, 43 ) -> None: 44 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 45 ### For some weird threading issue, 46 ### we can't use `isinstance` here. 47 if '_Plugin' in str(type(venv)): 48 self._venv = venv.name 49 self._activate = venv.activate_venv 50 self._deactivate = venv.deactivate_venv 51 self._kwargs = {} 52 else: 53 self._venv = venv 54 self._activate = activate_venv 55 self._deactivate = deactivate_venv 56 self._kwargs = {'venv': venv} 57 self._debug = debug 58 self._init_if_not_exists = init_if_not_exists 59 ### In case someone calls `deactivate()` before `activate()`. 60 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 61 62 63 def activate(self, debug: bool = False) -> bool: 64 """ 65 Activate this virtual environment. 66 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 67 will also be activated. 68 """ 69 from meerschaum.utils.venv import active_venvs, init_venv 70 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 71 try: 72 return self._activate( 73 debug=(debug or self._debug), 74 init_if_not_exists=self._init_if_not_exists, 75 **self._kwargs 76 ) 77 except OSError as e: 78 if self._init_if_not_exists: 79 if not init_venv(self._venv, force=True): 80 raise e 81 return self._activate( 82 debug=(debug or self._debug), 83 init_if_not_exists=self._init_if_not_exists, 84 **self._kwargs 85 ) 86 87 88 def deactivate(self, debug: bool = False) -> bool: 89 """ 90 Deactivate this virtual environment. 91 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 92 will also be deactivated. 93 """ 94 return self._deactivate(debug=(debug or self._debug), **self._kwargs) 95 96 97 @property 98 def target_path(self) -> pathlib.Path: 99 """ 100 Return the target site-packages path for this virtual environment. 101 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 102 (e.g. Python 3.10 and Python 3.7). 103 """ 104 from meerschaum.utils.venv import venv_target_path 105 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug) 106 107 108 @property 109 def root_path(self) -> pathlib.Path: 110 """ 111 Return the top-level path for this virtual environment. 112 """ 113 import meerschaum.config.paths as paths 114 if self._venv is None: 115 return self.target_path.parent 116 return paths.VIRTENV_RESOURCES_PATH / self._venv 117 118 119 def __enter__(self) -> None: 120 self.activate(debug=self._debug) 121 122 123 def __exit__(self, exc_type, exc_value, exc_traceback) -> None: 124 self.deactivate(debug=self._debug) 125 126 127 def __str__(self) -> str: 128 quote = "'" if self._venv is not None else "" 129 return "Venv(" + quote + str(self._venv) + quote + ")" 130 131 132 def __repr__(self) -> str: 133 return self.__str__()
Manage a virtual enviroment's activation status.
Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
... import pandas
>>> with Venv(Plugin('noaa')) as venv:
... import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
38 def __init__( 39 self, 40 venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm', 41 init_if_not_exists: bool = True, 42 debug: bool = False, 43 ) -> None: 44 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 45 ### For some weird threading issue, 46 ### we can't use `isinstance` here. 47 if '_Plugin' in str(type(venv)): 48 self._venv = venv.name 49 self._activate = venv.activate_venv 50 self._deactivate = venv.deactivate_venv 51 self._kwargs = {} 52 else: 53 self._venv = venv 54 self._activate = activate_venv 55 self._deactivate = deactivate_venv 56 self._kwargs = {'venv': venv} 57 self._debug = debug 58 self._init_if_not_exists = init_if_not_exists 59 ### In case someone calls `deactivate()` before `activate()`. 60 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
63 def activate(self, debug: bool = False) -> bool: 64 """ 65 Activate this virtual environment. 66 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 67 will also be activated. 68 """ 69 from meerschaum.utils.venv import active_venvs, init_venv 70 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 71 try: 72 return self._activate( 73 debug=(debug or self._debug), 74 init_if_not_exists=self._init_if_not_exists, 75 **self._kwargs 76 ) 77 except OSError as e: 78 if self._init_if_not_exists: 79 if not init_venv(self._venv, force=True): 80 raise e 81 return self._activate( 82 debug=(debug or self._debug), 83 init_if_not_exists=self._init_if_not_exists, 84 **self._kwargs 85 )
Activate this virtual environment.
If a meerschaum.plugins.Plugin was provided, its dependent virtual environments
will also be activated.
88 def deactivate(self, debug: bool = False) -> bool: 89 """ 90 Deactivate this virtual environment. 91 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 92 will also be deactivated. 93 """ 94 return self._deactivate(debug=(debug or self._debug), **self._kwargs)
Deactivate this virtual environment.
If a meerschaum.plugins.Plugin was provided, its dependent virtual environments
will also be deactivated.
97 @property 98 def target_path(self) -> pathlib.Path: 99 """ 100 Return the target site-packages path for this virtual environment. 101 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 102 (e.g. Python 3.10 and Python 3.7). 103 """ 104 from meerschaum.utils.venv import venv_target_path 105 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
Return the target site-packages path for this virtual environment.
A meerschaum.utils.venv.Venv may have one virtual environment per minor Python version
(e.g. Python 3.10 and Python 3.7).
108 @property 109 def root_path(self) -> pathlib.Path: 110 """ 111 Return the top-level path for this virtual environment. 112 """ 113 import meerschaum.config.paths as paths 114 if self._venv is None: 115 return self.target_path.parent 116 return paths.VIRTENV_RESOURCES_PATH / self._venv
Return the top-level path for this virtual environment.
39def activate_venv( 40 venv: Optional[str] = 'mrsm', 41 color: bool = True, 42 force: bool = False, 43 init_if_not_exists: bool = True, 44 debug: bool = False, 45 **kw 46) -> bool: 47 """ 48 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 49 50 Parameters 51 ---------- 52 venv: Optional[str], default 'mrsm' 53 The virtual environment to activate. 54 55 color: bool, default True 56 If `True`, include color in debug text. 57 58 init_if_not_exists: bool, default True 59 If `True`, create the virtual environment if it does not exist. 60 61 force: bool, default False 62 If `True`, do not exit early even if the venv is currently active. 63 64 debug: bool, default False 65 Verbosity toggle. 66 67 Returns 68 ------- 69 A bool indicating whether the virtual environment was successfully activated. 70 71 """ 72 thread_id = get_ident() 73 if active_venvs_order and active_venvs_order[0] == venv: 74 if not force: 75 return True 76 77 import sys 78 import os 79 if venv is not None and init_if_not_exists: 80 init_venv(venv=venv, debug=debug) 81 82 with LOCKS['active_venvs']: 83 if thread_id not in threads_active_venvs: 84 threads_active_venvs[thread_id] = {} 85 active_venvs.add(venv) 86 if venv not in threads_active_venvs[thread_id]: 87 threads_active_venvs[thread_id][venv] = 1 88 else: 89 threads_active_venvs[thread_id][venv] += 1 90 91 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 92 if not target_path.exists() and venv is not None and init_if_not_exists: 93 init_venv(venv=venv, force=True, debug=debug) 94 95 if not target_path.exists(): 96 if init_if_not_exists: 97 return False 98 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 99 100 target = target_path.as_posix() 101 102 if venv in active_venvs_order: 103 try: 104 sys.path.remove(target) 105 except Exception: 106 pass 107 try: 108 active_venvs_order.remove(venv) 109 except Exception: 110 pass 111 112 if venv is not None: 113 sys.path.insert(0, target) 114 else: 115 if sys.path and sys.path[0] in (os.getcwd(), ''): 116 sys.path.insert(1, target) 117 else: 118 sys.path.insert(0, target) 119 try: 120 active_venvs_order.insert(0, venv) 121 except Exception: 122 pass 123 124 return True
Create a virtual environment (if it doesn't exist) and add it to sys.path if necessary.
Parameters
- venv (Optional[str], default 'mrsm'): The virtual environment to activate.
- color (bool, default True):
If
True, include color in debug text. - init_if_not_exists (bool, default True):
If
True, create the virtual environment if it does not exist. - force (bool, default False):
If
True, do not exit early even if the venv is currently active. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment was successfully activated.
127def deactivate_venv( 128 venv: str = 'mrsm', 129 color: bool = True, 130 debug: bool = False, 131 previously_active_venvs: Union['set[str]', List[str], None] = None, 132 force: bool = False, 133 **kw 134) -> bool: 135 """ 136 Remove a virtual environment from `sys.path` (if it's been activated). 137 138 Parameters 139 ---------- 140 venv: str, default 'mrsm' 141 The virtual environment to deactivate. 142 143 color: bool, default True 144 If `True`, include color in debug text. 145 146 debug: bool, default False 147 Verbosity toggle. 148 149 previously_active_venvs: Union[Set[str], List[str], None] 150 If provided, skip deactivating if a virtual environment is in this iterable. 151 152 force: bool, default False 153 If `True`, forcibly deactivate the virtual environment. 154 This may cause issues with other threads, so be careful! 155 156 Returns 157 ------- 158 Return a bool indicating whether the virtual environment was successfully deactivated. 159 160 """ 161 import sys 162 thread_id = get_ident() 163 if venv is None: 164 if venv in active_venvs: 165 active_venvs.remove(venv) 166 return True 167 168 if previously_active_venvs and venv in previously_active_venvs and not force: 169 return True 170 171 with LOCKS['active_venvs']: 172 if venv in threads_active_venvs.get(thread_id, {}): 173 new_count = threads_active_venvs[thread_id][venv] - 1 174 if new_count > 0 and not force: 175 threads_active_venvs[thread_id][venv] = new_count 176 return True 177 else: 178 del threads_active_venvs[thread_id][venv] 179 180 if not force: 181 for other_thread_id, other_venvs in threads_active_venvs.items(): 182 if other_thread_id == thread_id: 183 continue 184 if venv in other_venvs: 185 return True 186 else: 187 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 188 for other_thread_id in to_delete: 189 del threads_active_venvs[other_thread_id] 190 191 if venv in active_venvs: 192 active_venvs.remove(venv) 193 194 if sys.path is None: 195 return False 196 197 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 198 with LOCKS['sys.path']: 199 if target in sys.path: 200 try: 201 sys.path.remove(target) 202 except Exception: 203 pass 204 try: 205 active_venvs_order.remove(venv) 206 except Exception: 207 pass 208 209 return True
Remove a virtual environment from sys.path (if it's been activated).
Parameters
- venv (str, default 'mrsm'): The virtual environment to deactivate.
- color (bool, default True):
If
True, include color in debug text. - debug (bool, default False): Verbosity toggle.
- previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
- force (bool, default False):
If
True, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
- Return a bool indicating whether the virtual environment was successfully deactivated.
849def get_module_venv(module) -> Union[str, None]: 850 """ 851 Return the virtual environment where an imported module is installed. 852 853 Parameters 854 ---------- 855 module: ModuleType 856 The imported module to inspect. 857 858 Returns 859 ------- 860 The name of a venv or `None`. 861 """ 862 import meerschaum.config.paths as paths 863 module_path = pathlib.Path(module.__file__).resolve() 864 try: 865 rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH) 866 except ValueError: 867 return None 868 869 return rel_path.as_posix().split('/', maxsplit=1)[0]
Return the virtual environment where an imported module is installed.
Parameters
- module (ModuleType): The imported module to inspect.
Returns
- The name of a venv or
None.
832def get_venvs() -> List[str]: 833 """ 834 Return a list of all the virtual environments. 835 """ 836 import os 837 import meerschaum.config.paths as paths 838 venvs = [] 839 for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH): 840 path = paths.VIRTENV_RESOURCES_PATH / filename 841 if not path.is_dir(): 842 continue 843 if not venv_exists(filename): 844 continue 845 venvs.append(filename) 846 return venvs
Return a list of all the virtual environments.
368def init_venv( 369 venv: str = 'mrsm', 370 verify: bool = True, 371 force: bool = False, 372 debug: bool = False, 373) -> bool: 374 """ 375 Initialize the virtual environment. 376 377 Parameters 378 ---------- 379 venv: str, default 'mrsm' 380 The name of the virtual environment to create. 381 382 verify: bool, default True 383 If `True`, verify that the virtual environment is in the expected state. 384 385 force: bool, default False 386 If `True`, recreate the virtual environment, even if already initalized. 387 388 Returns 389 ------- 390 A `bool` indicating success. 391 """ 392 if not force and venv in verified_venvs: 393 return True 394 if not force and venv_exists(venv, debug=debug): 395 if verify: 396 verify_venv(venv, debug=debug) 397 verified_venvs.add(venv) 398 return True 399 400 import io 401 from contextlib import redirect_stdout 402 import sys 403 import platform 404 import os 405 import shutil 406 import time 407 408 import meerschaum.config.paths as paths 409 from meerschaum._internal.static import STATIC_CONFIG 410 from meerschaum.utils.packages import is_uv_enabled 411 412 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 413 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 414 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 415 lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 416 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 417 418 def update_lock(active: bool): 419 try: 420 if not active: 421 if debug: 422 print(f"Releasing lock: '{lock_path}'") 423 lock_path.unlink() 424 else: 425 if debug: 426 print(f"Acquiring lock: '{lock_path}'") 427 lock_path.touch() 428 except Exception: 429 pass 430 431 def wait_for_lock(): 432 if platform.system() == 'Windows': 433 return 434 max_lock_seconds = 30.0 435 sleep_message_seconds = 5.0 436 step_sleep_seconds = 0.1 437 init_venv_check_start = time.perf_counter() 438 last_print = init_venv_check_start 439 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 440 if not lock_path.exists(): 441 break 442 443 now = time.perf_counter() 444 if debug or (now - last_print) > sleep_message_seconds: 445 print(f"Lock exists for venv '{venv}', sleeping...") 446 last_print = now 447 time.sleep(step_sleep_seconds) 448 update_lock(False) 449 450 if ( 451 not force 452 and venv == 'mrsm' 453 and os.environ.get(work_dir_env_var, None) is not None 454 and docker_home_venv_path.exists() 455 ): 456 wait_for_lock() 457 shutil.move(docker_home_venv_path, venv_path) 458 if verify: 459 verify_venv(venv, debug=debug) 460 verified_venvs.add(venv) 461 return True 462 463 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 464 global tried_virtualenv 465 try: 466 import venv as _venv 467 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 468 virtualenv = None 469 except ImportError: 470 _venv = None 471 uv = None 472 virtualenv = None 473 474 _venv_success = False 475 temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv) 476 ### NOTE: Disable site-packages movement for now. 477 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 478 479 wait_for_lock() 480 update_lock(True) 481 482 if rename_vtp: 483 if debug: 484 print(f"Moving '{vtp}' to '{temp_vtp}'...") 485 shutil.move(vtp, temp_vtp) 486 487 if uv is not None: 488 _venv_success = run_python_package( 489 'uv', 490 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 491 venv=None, 492 env=_get_pip_os_env(), 493 debug=debug, 494 ) == 0 495 496 if _venv is not None and not _venv_success: 497 f = io.StringIO() 498 with redirect_stdout(f): 499 _venv_success = run_python_package( 500 'venv', 501 [venv_path.as_posix()] + ( 502 ['--symlinks'] 503 if platform.system() != 'Windows' 504 else [] 505 ), 506 venv=None, debug=debug 507 ) == 0 508 if not _venv_success: 509 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 510 if not venv_exists(venv, debug=debug): 511 _venv = None 512 if not _venv_success: 513 virtualenv = attempt_import( 514 'virtualenv', 515 venv=None, 516 lazy=False, 517 install=(not tried_virtualenv), 518 warn=False, 519 check_update=False, 520 color=False, 521 debug=debug, 522 ) 523 if virtualenv is None: 524 print( 525 "Failed to import `venv` or `virtualenv`! " 526 + "Please install `virtualenv` via pip then restart Meerschaum." 527 ) 528 if rename_vtp and temp_vtp.exists(): 529 if debug: 530 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 531 shutil.move(temp_vtp, vtp) 532 update_lock(False) 533 return False 534 535 tried_virtualenv = True 536 try: 537 python_folder = ( 538 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 539 ) 540 dist_packages_path = ( 541 paths.VIRTENV_RESOURCES_PATH / 542 venv / 'local' / 'lib' / python_folder / 'dist-packages' 543 ) 544 local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 545 bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin' 546 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 547 if bin_path.exists(): 548 try: 549 shutil.rmtree(bin_path) 550 except Exception: 551 import traceback 552 traceback.print_exc() 553 virtualenv.cli_run([venv_path.as_posix()]) 554 if dist_packages_path.exists(): 555 vtp.mkdir(exist_ok=True, parents=True) 556 for file_path in dist_packages_path.glob('*'): 557 shutil.move(file_path, vtp) 558 shutil.rmtree(dist_packages_path) 559 # shutil.move(dist_packages_path, vtp) 560 bin_path.mkdir(exist_ok=True, parents=True) 561 for file_path in local_bin_path.glob('*'): 562 shutil.move(file_path, bin_path) 563 # shutil.move(local_bin_path, bin_path) 564 shutil.rmtree(local_bin_path) 565 566 except Exception: 567 import traceback 568 traceback.print_exc() 569 if rename_vtp and temp_vtp.exists(): 570 shutil.move(temp_vtp, vtp) 571 update_lock(False) 572 return False 573 if verify: 574 verify_venv(venv, debug=debug) 575 verified_venvs.add(venv) 576 577 if rename_vtp and temp_vtp.exists(): 578 if debug: 579 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 580 shutil.move(temp_vtp, vtp) 581 582 update_lock(False) 583 return True
Initialize the virtual environment.
Parameters
- venv (str, default 'mrsm'): The name of the virtual environment to create.
- verify (bool, default True):
If
True, verify that the virtual environment is in the expected state. - force (bool, default False):
If
True, recreate the virtual environment, even if already initalized.
Returns
- A
boolindicating success.
819def inside_venv() -> bool: 820 """ 821 Determine whether current Python interpreter is running inside a virtual environment. 822 """ 823 import sys 824 return ( 825 hasattr(sys, 'real_prefix') or ( 826 hasattr(sys, 'base_prefix') 827 and sys.base_prefix != sys.prefix 828 ) 829 )
Determine whether current Python interpreter is running inside a virtual environment.
212def is_venv_active( 213 venv: str = 'mrsm', 214 color : bool = True, 215 debug: bool = False 216) -> bool: 217 """ 218 Check if a virtual environment is active. 219 220 Parameters 221 ---------- 222 venv: str, default 'mrsm' 223 The virtual environment to check. 224 225 color: bool, default True 226 If `True`, include color in debug text. 227 228 debug: bool, default False 229 Verbosity toggle. 230 231 Returns 232 ------- 233 A bool indicating whether the virtual environment `venv` is active. 234 235 """ 236 return venv in active_venvs
Check if a virtual environment is active.
Parameters
- venv (str, default 'mrsm'): The virtual environment to check.
- color (bool, default True):
If
True, include color in debug text. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment
venvis active.
611def venv_exec( 612 code: str, 613 venv: Optional[str] = 'mrsm', 614 env: Optional[Dict[str, str]] = None, 615 with_extras: bool = False, 616 as_proc: bool = False, 617 capture_output: bool = True, 618 debug: bool = False, 619) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 620 """ 621 Execute Python code in a subprocess via a virtual environment's interpeter. 622 Return `True` if the code successfully executes, `False` on failure. 623 624 Parameters 625 ---------- 626 code: str 627 The Python code to excecute. 628 629 venv: str, default 'mrsm' 630 The virtual environment to use to get the path for the Python executable. 631 If `venv` is `None`, use the default `sys.executable` path. 632 633 env: Optional[Dict[str, str]], default None 634 Optionally specify the environment variables for the subprocess. 635 Defaults to `os.environ`. 636 637 with_extras: bool, default False 638 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 639 640 as_proc: bool, default False 641 If `True`, return the `subprocess.Popen` object instead of executing. 642 643 Returns 644 ------- 645 By default, return a bool indicating success. 646 If `as_proc` is `True`, return a `subprocess.Popen` object. 647 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 648 649 """ 650 import os 651 import subprocess 652 import platform 653 from meerschaum.utils.debug import dprint 654 from meerschaum.utils.process import _child_processes 655 656 executable = venv_executable(venv=venv) 657 cmd_list = [executable, '-c', code] 658 if env is None: 659 env = os.environ 660 if debug: 661 dprint(str(cmd_list)) 662 if not with_extras and not as_proc: 663 return subprocess.call(cmd_list, env=env) == 0 664 665 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 666 group_kwargs = ( 667 { 668 'preexec_fn': os.setsid, 669 } if platform.system() != 'Windows' 670 else { 671 'creationflags': CREATE_NEW_PROCESS_GROUP, 672 } 673 ) 674 process = subprocess.Popen( 675 cmd_list, 676 stdout=stdout, 677 stderr=stderr, 678 stdin=sys.stdin, 679 env=env, 680 **group_kwargs 681 ) 682 if as_proc: 683 _child_processes.append(process) 684 return process 685 stdout, stderr = process.communicate() 686 exit_code = process.returncode 687 return exit_code, stdout, stderr
Execute Python code in a subprocess via a virtual environment's interpeter.
Return True if the code successfully executes, False on failure.
Parameters
- code (str): The Python code to excecute.
- venv (str, default 'mrsm'):
The virtual environment to use to get the path for the Python executable.
If
venvisNone, use the defaultsys.executablepath. - env (Optional[Dict[str, str]], default None):
Optionally specify the environment variables for the subprocess.
Defaults to
os.environ. - with_extras (bool, default False):
If
True, return a tuple of the exit code, stdout bytes, and stderr bytes. - as_proc (bool, default False):
If
True, return thesubprocess.Popenobject instead of executing.
Returns
- By default, return a bool indicating success.
- If
as_procisTrue, return asubprocess.Popenobject. - If
with_extrasisTrue, return a tuple of the exit code, stdout bytes, and stderr bytes.
586def venv_executable(venv: Optional[str] = 'mrsm') -> str: 587 """ 588 The Python interpreter executable for a given virtual environment. 589 """ 590 import sys 591 import platform 592 import meerschaum.config.paths as paths 593 return ( 594 sys.executable if venv is None 595 else str( 596 paths.VIRTENV_RESOURCES_PATH 597 / venv 598 / ( 599 'bin' if platform.system() != 'Windows' 600 else 'Scripts' 601 ) / ( 602 'python' 603 + str(sys.version_info.major) 604 + '.' 605 + str(sys.version_info.minor) 606 ) 607 ) 608 )
The Python interpreter executable for a given virtual environment.
690def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 691 """ 692 Determine whether a virtual environment has been created. 693 """ 694 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 695 return target_path.exists()
Determine whether a virtual environment has been created.
698def venv_target_path( 699 venv: Union[str, None], 700 allow_nonexistent: bool = False, 701 debug: bool = False, 702) -> pathlib.Path: 703 """ 704 Return a virtual environment's site-package path. 705 706 Parameters 707 ---------- 708 venv: Union[str, None] 709 The virtual environment for which a path should be returned. 710 711 allow_nonexistent: bool, default False 712 If `True`, return a path even if it does not exist. 713 714 Returns 715 ------- 716 The `pathlib.Path` object for the virtual environment's path. 717 718 """ 719 import os 720 import sys 721 import platform 722 import site 723 724 import meerschaum.config.paths as paths 725 from meerschaum._internal.static import STATIC_CONFIG 726 727 ### Check sys.path for a user-writable site-packages directory. 728 if venv is None: 729 730 ### Return the known value for the portable environment. 731 environment_runtime = STATIC_CONFIG['environment']['runtime'] 732 if os.environ.get(environment_runtime, None) == 'portable': 733 python_version_folder = ( 734 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 735 ) 736 executable_path = pathlib.Path(sys.executable) 737 site_packages_path = ( 738 ( 739 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 740 ) if platform.system() != 'Windows' else ( 741 executable_path.parent / 'Lib' / 'site-packages' 742 ) 743 ) 744 if not site_packages_path.exists(): 745 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 746 return site_packages_path 747 748 if not inside_venv(): 749 user_site_packages = site.getusersitepackages() 750 if user_site_packages is None: 751 raise EnvironmentError("Could not determine user site packages.") 752 753 site_path = pathlib.Path(user_site_packages) 754 if not site_path.exists(): 755 756 ### Windows does not have `os.geteuid()`. 757 if platform.system() == 'Windows' or os.geteuid() != 0: 758 site_path.mkdir(parents=True, exist_ok=True) 759 return site_path 760 761 ### Allow for dist-level paths (running as root). 762 for possible_dist in site.getsitepackages(): 763 dist_path = pathlib.Path(possible_dist) 764 if not dist_path.exists(): 765 continue 766 return dist_path 767 768 raise EnvironmentError("Could not determine the dist-packages directory.") 769 770 return site_path 771 772 venv_root_path = ( 773 (paths.VIRTENV_RESOURCES_PATH / venv) 774 if venv is not None 775 else pathlib.Path(sys.prefix) 776 ) 777 target_path = venv_root_path 778 779 ### Ensure 'lib' or 'Lib' exists. 780 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 781 if not allow_nonexistent: 782 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 783 print(f"Failed to find lib directory for virtual environment '{venv}'.") 784 import traceback 785 traceback.print_stack() 786 sys.exit(1) 787 target_path = target_path / lib 788 789 ### Check if a 'python3.x' folder exists. 790 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 791 if target_path.exists(): 792 target_path = ( 793 (target_path / python_folder) 794 if python_folder in os.listdir(target_path) 795 else target_path 796 ) 797 else: 798 target_path = ( 799 (target_path / python_folder) 800 if platform.system() != 'Windows' 801 else target_path 802 ) 803 804 ### Ensure 'site-packages' exists. 805 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 806 target_path = target_path / 'site-packages' 807 else: 808 import traceback 809 traceback.print_stack() 810 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 811 print("This may be because you are using a different Python version.") 812 print("Try deleting the following directory and restarting Meerschaum:") 813 print(paths.VIRTENV_RESOURCES_PATH) 814 sys.exit(1) 815 816 return target_path
Return a virtual environment's site-package path.
Parameters
- venv (Union[str, None]): The virtual environment for which a path should be returned.
- allow_nonexistent (bool, default False):
If
True, return a path even if it does not exist.
Returns
- The
pathlib.Pathobject for the virtual environment's path.
240def verify_venv( 241 venv: str, 242 debug: bool = False, 243) -> None: 244 """ 245 Verify that the virtual environment matches the expected state. 246 """ 247 import platform 248 import os 249 import shutil 250 import sys 251 252 import meerschaum.config.paths as paths 253 from meerschaum.utils.process import run_process 254 from meerschaum.utils.misc import make_symlink, is_symlink 255 from meerschaum.utils.warnings import warn 256 257 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 258 bin_path = venv_path / ( 259 'bin' if platform.system() != 'Windows' else "Scripts" 260 ) 261 current_python_versioned_name = ( 262 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 263 + ('' if platform.system() != 'Windows' else '.exe') 264 ) 265 266 if not (bin_path / current_python_versioned_name).exists(): 267 init_venv(venv, verify=False, force=True, debug=debug) 268 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 269 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 270 if not current_python_in_venv_path.exists(): 271 if is_symlink(current_python_in_venv_path): 272 try: 273 current_python_in_venv_path.unlink() 274 except Exception as e: 275 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 276 try: 277 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 278 except Exception: 279 print( 280 f"Unable to create symlink {current_python_in_venv_path} " 281 + f"to {current_python_in_sys_path}." 282 ) 283 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 284 else: 285 files_to_inspect = [current_python_versioned_name] 286 287 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 288 """ 289 Return the version for the python binary at the given path. 290 """ 291 try: 292 ### It might be a broken symlink, so skip on errors. 293 if debug: 294 print(f"Getting python version for {python_path}") 295 proc = run_process( 296 [str(python_path), '-V'], 297 as_proc=True, 298 capture_output=True, 299 ) 300 stdout, stderr = proc.communicate(timeout=1.0) 301 except Exception as e: 302 ### E.g. the symlink may be broken. 303 if is_symlink(python_path): 304 try: 305 python_path.unlink() 306 except Exception as _e: 307 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 308 return None 309 return stdout.decode('utf-8').strip().replace('Python ', '') 310 311 ### Ensure the versions are symlinked correctly. 312 for filename in files_to_inspect: 313 if not filename.startswith('python'): 314 continue 315 python_path = bin_path / filename 316 version = get_python_version(python_path) 317 if version is None: 318 continue 319 try: 320 major_version = version.split('.', maxsplit=1)[0] 321 minor_version = version.split('.', maxsplit=2)[1] 322 except IndexError: 323 return 324 python_versioned_name = ( 325 'python' + major_version + '.' + minor_version 326 + ('' if platform.system() != 'Windows' else '.exe') 327 ) 328 329 ### E.g. python3.10 actually links to Python 3.10. 330 if filename == python_versioned_name: 331 try: 332 real_path = pathlib.Path(os.path.realpath(python_path)) 333 real_path_exists = real_path.exists() 334 except Exception: 335 real_path_exists = False 336 337 if not real_path_exists: 338 try: 339 python_path.unlink() 340 except Exception: 341 pass 342 init_venv(venv, verify=False, force=True, debug=debug) 343 if not python_path.exists(): 344 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 345 346 if python_path == real_path: 347 continue 348 349 try: 350 python_path.unlink() 351 except Exception: 352 pass 353 success, msg = make_symlink(real_path, python_path) 354 if not success: 355 warn(msg, color=False) 356 continue 357 358 python_versioned_path = bin_path / python_versioned_name 359 if python_versioned_path.exists(): 360 ### Avoid circular symlinks. 361 if get_python_version(python_versioned_path) == version: 362 continue 363 python_versioned_path.unlink() 364 shutil.move(python_path, python_versioned_path)
Verify that the virtual environment matches the expected state.