meerschaum.utils.venv
Manage virtual environments.
1#! /usr/bin/env python3 2# -*- coding: utf-8 -*- 3# vim:fenc=utf-8 4 5""" 6Manage virtual environments. 7""" 8 9from __future__ import annotations 10 11import sys 12import pathlib 13 14from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple 15from meerschaum.utils.threading import RLock, get_ident 16 17__all__ = sorted([ 18 'activate_venv', 'deactivate_venv', 'init_venv', 19 'inside_venv', 'is_venv_active', 'venv_exec', 20 'venv_executable', 'venv_exists', 'venv_target_path', 21 'Venv', 'get_venvs', 'verify_venv', 'get_module_venv', 22]) 23__pdoc__ = {'Venv': True} 24 25LOCKS = { 26 'sys.path': RLock(), 27 'active_venvs': RLock(), 28 'venvs_active_counts': RLock(), 29} 30 31active_venvs = set() 32active_venvs_counts: Dict[str, int] = {} 33active_venvs_order: List[Optional[str]] = [] 34threads_active_venvs: Dict[int, 'set[str]'] = {} 35CREATE_NEW_PROCESS_GROUP = 0x00000200 36 37 38def activate_venv( 39 venv: Optional[str] = 'mrsm', 40 color: bool = True, 41 force: bool = False, 42 init_if_not_exists: bool = True, 43 debug: bool = False, 44 **kw 45) -> bool: 46 """ 47 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 48 49 Parameters 50 ---------- 51 venv: Optional[str], default 'mrsm' 52 The virtual environment to activate. 53 54 color: bool, default True 55 If `True`, include color in debug text. 56 57 init_if_not_exists: bool, default True 58 If `True`, create the virtual environment if it does not exist. 59 60 force: bool, default False 61 If `True`, do not exit early even if the venv is currently active. 62 63 debug: bool, default False 64 Verbosity toggle. 65 66 Returns 67 ------- 68 A bool indicating whether the virtual environment was successfully activated. 69 70 """ 71 thread_id = get_ident() 72 import sys 73 import os 74 if venv is not None and init_if_not_exists: 75 init_venv(venv=venv, debug=debug) 76 77 with LOCKS['active_venvs']: 78 if thread_id not in threads_active_venvs: 79 threads_active_venvs[thread_id] = {} 80 active_venvs.add(venv) 81 if venv not in threads_active_venvs[thread_id]: 82 threads_active_venvs[thread_id][venv] = 1 83 else: 84 threads_active_venvs[thread_id][venv] += 1 85 86 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 87 if not target_path.exists() and venv is not None and init_if_not_exists: 88 init_venv(venv=venv, force=True, debug=debug) 89 90 if not target_path.exists(): 91 if init_if_not_exists: 92 return False 93 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 94 95 target = target_path.as_posix() 96 if ( 97 active_venvs_order 98 and active_venvs_order[0] == venv 99 and target in sys.path 100 and not force 101 ): 102 return True 103 104 if venv in active_venvs_order: 105 try: 106 sys.path.remove(target) 107 except Exception: 108 pass 109 try: 110 active_venvs_order.remove(venv) 111 except Exception: 112 pass 113 114 if venv is not None: 115 sys.path.insert(0, target) 116 else: 117 if sys.path and sys.path[0] in (os.getcwd(), ''): 118 sys.path.insert(1, target) 119 else: 120 sys.path.insert(0, target) 121 try: 122 active_venvs_order.insert(0, venv) 123 except Exception: 124 pass 125 126 return True 127 128 129def deactivate_venv( 130 venv: str = 'mrsm', 131 color: bool = True, 132 debug: bool = False, 133 previously_active_venvs: Union['set[str]', List[str], None] = None, 134 force: bool = False, 135 **kw 136) -> bool: 137 """ 138 Remove a virtual environment from `sys.path` (if it's been activated). 139 140 Parameters 141 ---------- 142 venv: str, default 'mrsm' 143 The virtual environment to deactivate. 144 145 color: bool, default True 146 If `True`, include color in debug text. 147 148 debug: bool, default False 149 Verbosity toggle. 150 151 previously_active_venvs: Union[Set[str], List[str], None] 152 If provided, skip deactivating if a virtual environment is in this iterable. 153 154 force: bool, default False 155 If `True`, forcibly deactivate the virtual environment. 156 This may cause issues with other threads, so be careful! 157 158 Returns 159 ------- 160 Return a bool indicating whether the virtual environment was successfully deactivated. 161 162 """ 163 import sys 164 thread_id = get_ident() 165 if venv is None: 166 if venv in active_venvs: 167 active_venvs.remove(venv) 168 return True 169 170 if previously_active_venvs and venv in previously_active_venvs and not force: 171 return True 172 173 with LOCKS['active_venvs']: 174 if venv in threads_active_venvs.get(thread_id, {}): 175 new_count = threads_active_venvs[thread_id][venv] - 1 176 if new_count > 0 and not force: 177 threads_active_venvs[thread_id][venv] = new_count 178 return True 179 else: 180 del threads_active_venvs[thread_id][venv] 181 182 if not force: 183 for other_thread_id, other_venvs in threads_active_venvs.items(): 184 if other_thread_id == thread_id: 185 continue 186 if venv in other_venvs: 187 return True 188 else: 189 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 190 for other_thread_id in to_delete: 191 del threads_active_venvs[other_thread_id] 192 193 if venv in active_venvs: 194 active_venvs.remove(venv) 195 196 if sys.path is None: 197 return False 198 199 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 200 with LOCKS['sys.path']: 201 if target in sys.path: 202 try: 203 sys.path.remove(target) 204 except Exception: 205 pass 206 try: 207 active_venvs_order.remove(venv) 208 except Exception: 209 pass 210 211 return True 212 213 214def is_venv_active( 215 venv: str = 'mrsm', 216 color : bool = True, 217 debug: bool = False 218) -> bool: 219 """ 220 Check if a virtual environment is active. 221 222 Parameters 223 ---------- 224 venv: str, default 'mrsm' 225 The virtual environment to check. 226 227 color: bool, default True 228 If `True`, include color in debug text. 229 230 debug: bool, default False 231 Verbosity toggle. 232 233 Returns 234 ------- 235 A bool indicating whether the virtual environment `venv` is active. 236 237 """ 238 return venv in active_venvs 239 240 241verified_venvs = set() 242def verify_venv( 243 venv: str, 244 debug: bool = False, 245) -> None: 246 """ 247 Verify that the virtual environment matches the expected state. 248 """ 249 import platform 250 import os 251 import shutil 252 import sys 253 254 import meerschaum.config.paths as paths 255 from meerschaum.utils.process import run_process 256 from meerschaum.utils.misc import make_symlink, is_symlink 257 from meerschaum.utils.warnings import warn 258 259 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 260 bin_path = venv_path / ( 261 'bin' if platform.system() != 'Windows' else "Scripts" 262 ) 263 current_python_versioned_name = ( 264 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 265 + ('' if platform.system() != 'Windows' else '.exe') 266 ) 267 268 if not (bin_path / current_python_versioned_name).exists(): 269 init_venv(venv, verify=False, force=True, debug=debug) 270 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 271 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 272 if not current_python_in_venv_path.exists(): 273 if is_symlink(current_python_in_venv_path): 274 try: 275 current_python_in_venv_path.unlink() 276 except Exception as e: 277 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 278 try: 279 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 280 except Exception: 281 print( 282 f"Unable to create symlink {current_python_in_venv_path} " 283 + f"to {current_python_in_sys_path}." 284 ) 285 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 286 else: 287 files_to_inspect = [current_python_versioned_name] 288 289 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 290 """ 291 Return the version for the python binary at the given path. 292 """ 293 try: 294 ### It might be a broken symlink, so skip on errors. 295 if debug: 296 print(f"Getting python version for {python_path}") 297 proc = run_process( 298 [str(python_path), '-V'], 299 as_proc=True, 300 capture_output=True, 301 ) 302 stdout, stderr = proc.communicate(timeout=1.0) 303 except Exception as e: 304 ### E.g. the symlink may be broken. 305 if is_symlink(python_path): 306 try: 307 python_path.unlink() 308 except Exception as _e: 309 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 310 return None 311 return stdout.decode('utf-8').strip().replace('Python ', '') 312 313 ### Ensure the versions are symlinked correctly. 314 for filename in files_to_inspect: 315 if not filename.startswith('python'): 316 continue 317 python_path = bin_path / filename 318 version = get_python_version(python_path) 319 if version is None: 320 continue 321 try: 322 major_version = version.split('.', maxsplit=1)[0] 323 minor_version = version.split('.', maxsplit=2)[1] 324 except IndexError: 325 return 326 python_versioned_name = ( 327 'python' + major_version + '.' + minor_version 328 + ('' if platform.system() != 'Windows' else '.exe') 329 ) 330 331 ### E.g. python3.10 actually links to Python 3.10. 332 if filename == python_versioned_name: 333 try: 334 real_path = pathlib.Path(os.path.realpath(python_path)) 335 real_path_exists = real_path.exists() 336 except Exception: 337 real_path_exists = False 338 339 if not real_path_exists: 340 try: 341 python_path.unlink() 342 except Exception: 343 pass 344 init_venv(venv, verify=False, force=True, debug=debug) 345 if not python_path.exists(): 346 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 347 348 if python_path == real_path: 349 continue 350 351 try: 352 python_path.unlink() 353 except Exception: 354 pass 355 success, msg = make_symlink(real_path, python_path) 356 if not success: 357 warn(msg, color=False) 358 continue 359 360 python_versioned_path = bin_path / python_versioned_name 361 if python_versioned_path.exists(): 362 ### Avoid circular symlinks. 363 if get_python_version(python_versioned_path) == version: 364 continue 365 python_versioned_path.unlink() 366 shutil.move(python_path, python_versioned_path) 367 368 369tried_virtualenv = False 370def init_venv( 371 venv: str = 'mrsm', 372 verify: bool = True, 373 force: bool = False, 374 debug: bool = False, 375) -> bool: 376 """ 377 Initialize the virtual environment. 378 379 Parameters 380 ---------- 381 venv: str, default 'mrsm' 382 The name of the virtual environment to create. 383 384 verify: bool, default True 385 If `True`, verify that the virtual environment is in the expected state. 386 387 force: bool, default False 388 If `True`, recreate the virtual environment, even if already initalized. 389 390 Returns 391 ------- 392 A `bool` indicating success. 393 """ 394 if not force and venv in verified_venvs: 395 return True 396 if not force and venv_exists(venv, debug=debug): 397 if verify: 398 verify_venv(venv, debug=debug) 399 verified_venvs.add(venv) 400 return True 401 402 import io 403 from contextlib import redirect_stdout 404 import sys 405 import platform 406 import os 407 import shutil 408 import time 409 410 import meerschaum.config.paths as paths 411 from meerschaum._internal.static import STATIC_CONFIG 412 from meerschaum.utils.packages import is_uv_enabled 413 414 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 415 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 416 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 417 lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 418 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 419 420 def update_lock(active: bool): 421 try: 422 if not active: 423 if debug: 424 print(f"Releasing lock: '{lock_path}'") 425 lock_path.unlink() 426 else: 427 if debug: 428 print(f"Acquiring lock: '{lock_path}'") 429 lock_path.touch() 430 except Exception: 431 pass 432 433 def wait_for_lock(): 434 if platform.system() == 'Windows': 435 return 436 max_lock_seconds = 30.0 437 sleep_message_seconds = 5.0 438 step_sleep_seconds = 0.1 439 init_venv_check_start = time.perf_counter() 440 last_print = init_venv_check_start 441 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 442 if not lock_path.exists(): 443 break 444 445 now = time.perf_counter() 446 if debug or (now - last_print) > sleep_message_seconds: 447 print(f"Lock exists for venv '{venv}', sleeping...") 448 last_print = now 449 time.sleep(step_sleep_seconds) 450 update_lock(False) 451 452 if ( 453 not force 454 and venv == 'mrsm' 455 and os.environ.get(work_dir_env_var, None) is not None 456 and docker_home_venv_path.exists() 457 ): 458 wait_for_lock() 459 shutil.move(docker_home_venv_path, venv_path) 460 if verify: 461 verify_venv(venv, debug=debug) 462 verified_venvs.add(venv) 463 return True 464 465 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 466 global tried_virtualenv 467 try: 468 import venv as _venv 469 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 470 virtualenv = None 471 except ImportError: 472 _venv = None 473 uv = None 474 virtualenv = None 475 476 _venv_success = False 477 temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv) 478 ### NOTE: Disable site-packages movement for now. 479 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 480 481 wait_for_lock() 482 update_lock(True) 483 484 if rename_vtp: 485 if debug: 486 print(f"Moving '{vtp}' to '{temp_vtp}'...") 487 shutil.move(vtp, temp_vtp) 488 489 if uv is not None: 490 _venv_success = run_python_package( 491 'uv', 492 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 493 venv=None, 494 env=_get_pip_os_env(), 495 debug=debug, 496 ) == 0 497 498 if _venv is not None and not _venv_success: 499 f = io.StringIO() 500 with redirect_stdout(f): 501 _venv_success = run_python_package( 502 'venv', 503 [venv_path.as_posix()] + ( 504 ['--symlinks'] 505 if platform.system() != 'Windows' 506 else [] 507 ), 508 venv=None, debug=debug 509 ) == 0 510 if not _venv_success: 511 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 512 if not venv_exists(venv, debug=debug): 513 _venv = None 514 if not _venv_success: 515 virtualenv = attempt_import( 516 'virtualenv', 517 venv=None, 518 lazy=False, 519 install=(not tried_virtualenv), 520 warn=False, 521 check_update=False, 522 color=False, 523 debug=debug, 524 ) 525 if virtualenv is None: 526 print( 527 "Failed to import `venv` or `virtualenv`! " 528 + "Please install `virtualenv` via pip then restart Meerschaum." 529 ) 530 if rename_vtp and temp_vtp.exists(): 531 if debug: 532 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 533 shutil.move(temp_vtp, vtp) 534 update_lock(False) 535 return False 536 537 tried_virtualenv = True 538 try: 539 python_folder = ( 540 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 541 ) 542 dist_packages_path = ( 543 paths.VIRTENV_RESOURCES_PATH / 544 venv / 'local' / 'lib' / python_folder / 'dist-packages' 545 ) 546 local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 547 bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin' 548 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 549 if bin_path.exists(): 550 try: 551 shutil.rmtree(bin_path) 552 except Exception: 553 import traceback 554 traceback.print_exc() 555 virtualenv.cli_run([venv_path.as_posix()]) 556 if dist_packages_path.exists(): 557 vtp.mkdir(exist_ok=True, parents=True) 558 for file_path in dist_packages_path.glob('*'): 559 shutil.move(file_path, vtp) 560 shutil.rmtree(dist_packages_path) 561 # shutil.move(dist_packages_path, vtp) 562 bin_path.mkdir(exist_ok=True, parents=True) 563 for file_path in local_bin_path.glob('*'): 564 shutil.move(file_path, bin_path) 565 # shutil.move(local_bin_path, bin_path) 566 shutil.rmtree(local_bin_path) 567 568 except Exception: 569 import traceback 570 traceback.print_exc() 571 if rename_vtp and temp_vtp.exists(): 572 shutil.move(temp_vtp, vtp) 573 update_lock(False) 574 return False 575 if verify: 576 verify_venv(venv, debug=debug) 577 verified_venvs.add(venv) 578 579 if rename_vtp and temp_vtp.exists(): 580 if debug: 581 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 582 shutil.move(temp_vtp, vtp) 583 584 update_lock(False) 585 return True 586 587 588def venv_executable(venv: Optional[str] = 'mrsm') -> str: 589 """ 590 The Python interpreter executable for a given virtual environment. 591 """ 592 import sys 593 import platform 594 import meerschaum.config.paths as paths 595 return ( 596 sys.executable if venv is None 597 else str( 598 paths.VIRTENV_RESOURCES_PATH 599 / venv 600 / ( 601 'bin' if platform.system() != 'Windows' 602 else 'Scripts' 603 ) / ( 604 'python' 605 + str(sys.version_info.major) 606 + '.' 607 + str(sys.version_info.minor) 608 ) 609 ) 610 ) 611 612 613def venv_exec( 614 code: str, 615 venv: Optional[str] = 'mrsm', 616 env: Optional[Dict[str, str]] = None, 617 with_extras: bool = False, 618 as_proc: bool = False, 619 capture_output: bool = True, 620 debug: bool = False, 621) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 622 """ 623 Execute Python code in a subprocess via a virtual environment's interpeter. 624 Return `True` if the code successfully executes, `False` on failure. 625 626 Parameters 627 ---------- 628 code: str 629 The Python code to excecute. 630 631 venv: str, default 'mrsm' 632 The virtual environment to use to get the path for the Python executable. 633 If `venv` is `None`, use the default `sys.executable` path. 634 635 env: Optional[Dict[str, str]], default None 636 Optionally specify the environment variables for the subprocess. 637 Defaults to `os.environ`. 638 639 with_extras: bool, default False 640 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 641 642 as_proc: bool, default False 643 If `True`, return the `subprocess.Popen` object instead of executing. 644 645 Returns 646 ------- 647 By default, return a bool indicating success. 648 If `as_proc` is `True`, return a `subprocess.Popen` object. 649 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 650 651 """ 652 import os 653 import subprocess 654 import platform 655 from meerschaum.utils.debug import dprint 656 from meerschaum.utils.process import _child_processes 657 658 executable = venv_executable(venv=venv) 659 cmd_list = [executable, '-c', code] 660 if env is None: 661 env = os.environ 662 if debug: 663 dprint(str(cmd_list)) 664 if not with_extras and not as_proc: 665 return subprocess.call(cmd_list, env=env) == 0 666 667 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 668 group_kwargs = ( 669 { 670 'preexec_fn': os.setsid, 671 } if platform.system() != 'Windows' 672 else { 673 'creationflags': CREATE_NEW_PROCESS_GROUP, 674 } 675 ) 676 process = subprocess.Popen( 677 cmd_list, 678 stdout=stdout, 679 stderr=stderr, 680 stdin=sys.stdin, 681 env=env, 682 **group_kwargs 683 ) 684 if as_proc: 685 _child_processes.append(process) 686 return process 687 stdout, stderr = process.communicate() 688 exit_code = process.returncode 689 return exit_code, stdout, stderr 690 691 692def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 693 """ 694 Determine whether a virtual environment has been created. 695 """ 696 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 697 return target_path.exists() 698 699 700def venv_target_path( 701 venv: Union[str, None], 702 allow_nonexistent: bool = False, 703 debug: bool = False, 704) -> pathlib.Path: 705 """ 706 Return a virtual environment's site-package path. 707 708 Parameters 709 ---------- 710 venv: Union[str, None] 711 The virtual environment for which a path should be returned. 712 713 allow_nonexistent: bool, default False 714 If `True`, return a path even if it does not exist. 715 716 Returns 717 ------- 718 The `pathlib.Path` object for the virtual environment's path. 719 720 """ 721 import os 722 import sys 723 import platform 724 import site 725 726 import meerschaum.config.paths as paths 727 from meerschaum._internal.static import STATIC_CONFIG 728 729 ### Check sys.path for a user-writable site-packages directory. 730 if venv is None: 731 732 ### Return the known value for the portable environment. 733 environment_runtime = STATIC_CONFIG['environment']['runtime'] 734 if os.environ.get(environment_runtime, None) == 'portable': 735 python_version_folder = ( 736 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 737 ) 738 executable_path = pathlib.Path(sys.executable) 739 site_packages_path = ( 740 ( 741 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 742 ) if platform.system() != 'Windows' else ( 743 executable_path.parent / 'Lib' / 'site-packages' 744 ) 745 ) 746 if not site_packages_path.exists(): 747 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 748 return site_packages_path 749 750 if not inside_venv(): 751 user_site_packages = site.getusersitepackages() 752 if user_site_packages is None: 753 raise EnvironmentError("Could not determine user site packages.") 754 755 site_path = pathlib.Path(user_site_packages) 756 if not site_path.exists(): 757 758 ### Windows does not have `os.geteuid()`. 759 if platform.system() == 'Windows' or os.geteuid() != 0: 760 site_path.mkdir(parents=True, exist_ok=True) 761 return site_path 762 763 ### Allow for dist-level paths (running as root). 764 for possible_dist in site.getsitepackages(): 765 dist_path = pathlib.Path(possible_dist) 766 if not dist_path.exists(): 767 continue 768 return dist_path 769 770 raise EnvironmentError("Could not determine the dist-packages directory.") 771 772 return site_path 773 774 venv_root_path = ( 775 (paths.VIRTENV_RESOURCES_PATH / venv) 776 if venv is not None 777 else pathlib.Path(sys.prefix) 778 ) 779 target_path = venv_root_path 780 781 ### Ensure 'lib' or 'Lib' exists. 782 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 783 if not allow_nonexistent: 784 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 785 print(f"Failed to find lib directory for virtual environment '{venv}'.") 786 import traceback 787 traceback.print_stack() 788 sys.exit(1) 789 target_path = target_path / lib 790 791 ### Check if a 'python3.x' folder exists. 792 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 793 if target_path.exists(): 794 target_path = ( 795 (target_path / python_folder) 796 if python_folder in os.listdir(target_path) 797 else target_path 798 ) 799 else: 800 target_path = ( 801 (target_path / python_folder) 802 if platform.system() != 'Windows' 803 else target_path 804 ) 805 806 ### Ensure 'site-packages' exists. 807 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 808 target_path = target_path / 'site-packages' 809 else: 810 import traceback 811 traceback.print_stack() 812 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 813 print("This may be because you are using a different Python version.") 814 print("Try deleting the following directory and restarting Meerschaum:") 815 print(paths.VIRTENV_RESOURCES_PATH) 816 sys.exit(1) 817 818 return target_path 819 820 821def inside_venv() -> bool: 822 """ 823 Determine whether current Python interpreter is running inside a virtual environment. 824 """ 825 import sys 826 return ( 827 hasattr(sys, 'real_prefix') or ( 828 hasattr(sys, 'base_prefix') 829 and sys.base_prefix != sys.prefix 830 ) 831 ) 832 833 834def get_venvs() -> List[str]: 835 """ 836 Return a list of all the virtual environments. 837 """ 838 import os 839 import meerschaum.config.paths as paths 840 venvs = [] 841 for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH): 842 path = paths.VIRTENV_RESOURCES_PATH / filename 843 if not path.is_dir(): 844 continue 845 if not venv_exists(filename): 846 continue 847 venvs.append(filename) 848 return venvs 849 850 851def get_module_venv(module) -> Union[str, None]: 852 """ 853 Return the virtual environment where an imported module is installed. 854 855 Parameters 856 ---------- 857 module: ModuleType 858 The imported module to inspect. 859 860 Returns 861 ------- 862 The name of a venv or `None`. 863 """ 864 import meerschaum.config.paths as paths 865 module_path = pathlib.Path(module.__file__).resolve() 866 try: 867 rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH) 868 except ValueError: 869 return None 870 871 return rel_path.as_posix().split('/', maxsplit=1)[0] 872 873 874from meerschaum.utils.venv._Venv import Venv
19class Venv: 20 """ 21 Manage a virtual enviroment's activation status. 22 23 Examples 24 -------- 25 >>> from meerschaum.plugins import Plugin 26 >>> with Venv('mrsm') as venv: 27 ... import pandas 28 >>> with Venv(Plugin('noaa')) as venv: 29 ... import requests 30 >>> venv = Venv('mrsm') 31 >>> venv.activate() 32 True 33 >>> venv.deactivate() 34 True 35 >>> 36 """ 37 38 def __init__( 39 self, 40 venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm', 41 init_if_not_exists: bool = True, 42 debug: bool = False, 43 ) -> None: 44 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 45 ### For some weird threading issue, 46 ### we can't use `isinstance` here. 47 if '_Plugin' in str(type(venv)): 48 self._venv = venv.name 49 self._activate = venv.activate_venv 50 self._deactivate = venv.deactivate_venv 51 self._kwargs = {} 52 else: 53 self._venv = venv 54 self._activate = activate_venv 55 self._deactivate = deactivate_venv 56 self._kwargs = {'venv': venv} 57 self._debug = debug 58 self._init_if_not_exists = init_if_not_exists 59 ### In case someone calls `deactivate()` before `activate()`. 60 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 61 62 63 def activate(self, debug: bool = False) -> bool: 64 """ 65 Activate this virtual environment. 66 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 67 will also be activated. 68 """ 69 from meerschaum.utils.venv import active_venvs, init_venv 70 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 71 try: 72 return self._activate( 73 debug=(debug or self._debug), 74 init_if_not_exists=self._init_if_not_exists, 75 **self._kwargs 76 ) 77 except OSError as e: 78 if self._init_if_not_exists: 79 if not init_venv(self._venv, force=True): 80 raise e 81 return self._activate( 82 debug=(debug or self._debug), 83 init_if_not_exists=self._init_if_not_exists, 84 **self._kwargs 85 ) 86 87 88 def deactivate(self, debug: bool = False) -> bool: 89 """ 90 Deactivate this virtual environment. 91 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 92 will also be deactivated. 93 """ 94 return self._deactivate(debug=(debug or self._debug), **self._kwargs) 95 96 97 @property 98 def target_path(self) -> pathlib.Path: 99 """ 100 Return the target site-packages path for this virtual environment. 101 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 102 (e.g. Python 3.10 and Python 3.7). 103 """ 104 from meerschaum.utils.venv import venv_target_path 105 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug) 106 107 108 @property 109 def root_path(self) -> pathlib.Path: 110 """ 111 Return the top-level path for this virtual environment. 112 """ 113 import meerschaum.config.paths as paths 114 if self._venv is None: 115 return self.target_path.parent 116 return paths.VIRTENV_RESOURCES_PATH / self._venv 117 118 119 def __enter__(self) -> None: 120 self.activate(debug=self._debug) 121 122 123 def __exit__(self, exc_type, exc_value, exc_traceback) -> None: 124 self.deactivate(debug=self._debug) 125 126 127 def __str__(self) -> str: 128 quote = "'" if self._venv is not None else "" 129 return "Venv(" + quote + str(self._venv) + quote + ")" 130 131 132 def __repr__(self) -> str: 133 return self.__str__()
Manage a virtual enviroment's activation status.
Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
... import pandas
>>> with Venv(Plugin('noaa')) as venv:
... import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
38 def __init__( 39 self, 40 venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm', 41 init_if_not_exists: bool = True, 42 debug: bool = False, 43 ) -> None: 44 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 45 ### For some weird threading issue, 46 ### we can't use `isinstance` here. 47 if '_Plugin' in str(type(venv)): 48 self._venv = venv.name 49 self._activate = venv.activate_venv 50 self._deactivate = venv.deactivate_venv 51 self._kwargs = {} 52 else: 53 self._venv = venv 54 self._activate = activate_venv 55 self._deactivate = deactivate_venv 56 self._kwargs = {'venv': venv} 57 self._debug = debug 58 self._init_if_not_exists = init_if_not_exists 59 ### In case someone calls `deactivate()` before `activate()`. 60 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
63 def activate(self, debug: bool = False) -> bool: 64 """ 65 Activate this virtual environment. 66 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 67 will also be activated. 68 """ 69 from meerschaum.utils.venv import active_venvs, init_venv 70 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 71 try: 72 return self._activate( 73 debug=(debug or self._debug), 74 init_if_not_exists=self._init_if_not_exists, 75 **self._kwargs 76 ) 77 except OSError as e: 78 if self._init_if_not_exists: 79 if not init_venv(self._venv, force=True): 80 raise e 81 return self._activate( 82 debug=(debug or self._debug), 83 init_if_not_exists=self._init_if_not_exists, 84 **self._kwargs 85 )
Activate this virtual environment.
If a meerschaum.plugins.Plugin was provided, its dependent virtual environments
will also be activated.
88 def deactivate(self, debug: bool = False) -> bool: 89 """ 90 Deactivate this virtual environment. 91 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 92 will also be deactivated. 93 """ 94 return self._deactivate(debug=(debug or self._debug), **self._kwargs)
Deactivate this virtual environment.
If a meerschaum.plugins.Plugin was provided, its dependent virtual environments
will also be deactivated.
97 @property 98 def target_path(self) -> pathlib.Path: 99 """ 100 Return the target site-packages path for this virtual environment. 101 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 102 (e.g. Python 3.10 and Python 3.7). 103 """ 104 from meerschaum.utils.venv import venv_target_path 105 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
Return the target site-packages path for this virtual environment.
A meerschaum.utils.venv.Venv may have one virtual environment per minor Python version
(e.g. Python 3.10 and Python 3.7).
108 @property 109 def root_path(self) -> pathlib.Path: 110 """ 111 Return the top-level path for this virtual environment. 112 """ 113 import meerschaum.config.paths as paths 114 if self._venv is None: 115 return self.target_path.parent 116 return paths.VIRTENV_RESOURCES_PATH / self._venv
Return the top-level path for this virtual environment.
39def activate_venv( 40 venv: Optional[str] = 'mrsm', 41 color: bool = True, 42 force: bool = False, 43 init_if_not_exists: bool = True, 44 debug: bool = False, 45 **kw 46) -> bool: 47 """ 48 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 49 50 Parameters 51 ---------- 52 venv: Optional[str], default 'mrsm' 53 The virtual environment to activate. 54 55 color: bool, default True 56 If `True`, include color in debug text. 57 58 init_if_not_exists: bool, default True 59 If `True`, create the virtual environment if it does not exist. 60 61 force: bool, default False 62 If `True`, do not exit early even if the venv is currently active. 63 64 debug: bool, default False 65 Verbosity toggle. 66 67 Returns 68 ------- 69 A bool indicating whether the virtual environment was successfully activated. 70 71 """ 72 thread_id = get_ident() 73 import sys 74 import os 75 if venv is not None and init_if_not_exists: 76 init_venv(venv=venv, debug=debug) 77 78 with LOCKS['active_venvs']: 79 if thread_id not in threads_active_venvs: 80 threads_active_venvs[thread_id] = {} 81 active_venvs.add(venv) 82 if venv not in threads_active_venvs[thread_id]: 83 threads_active_venvs[thread_id][venv] = 1 84 else: 85 threads_active_venvs[thread_id][venv] += 1 86 87 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 88 if not target_path.exists() and venv is not None and init_if_not_exists: 89 init_venv(venv=venv, force=True, debug=debug) 90 91 if not target_path.exists(): 92 if init_if_not_exists: 93 return False 94 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 95 96 target = target_path.as_posix() 97 if ( 98 active_venvs_order 99 and active_venvs_order[0] == venv 100 and target in sys.path 101 and not force 102 ): 103 return True 104 105 if venv in active_venvs_order: 106 try: 107 sys.path.remove(target) 108 except Exception: 109 pass 110 try: 111 active_venvs_order.remove(venv) 112 except Exception: 113 pass 114 115 if venv is not None: 116 sys.path.insert(0, target) 117 else: 118 if sys.path and sys.path[0] in (os.getcwd(), ''): 119 sys.path.insert(1, target) 120 else: 121 sys.path.insert(0, target) 122 try: 123 active_venvs_order.insert(0, venv) 124 except Exception: 125 pass 126 127 return True
Create a virtual environment (if it doesn't exist) and add it to sys.path if necessary.
Parameters
- venv (Optional[str], default 'mrsm'): The virtual environment to activate.
- color (bool, default True):
If
True, include color in debug text. - init_if_not_exists (bool, default True):
If
True, create the virtual environment if it does not exist. - force (bool, default False):
If
True, do not exit early even if the venv is currently active. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment was successfully activated.
130def deactivate_venv( 131 venv: str = 'mrsm', 132 color: bool = True, 133 debug: bool = False, 134 previously_active_venvs: Union['set[str]', List[str], None] = None, 135 force: bool = False, 136 **kw 137) -> bool: 138 """ 139 Remove a virtual environment from `sys.path` (if it's been activated). 140 141 Parameters 142 ---------- 143 venv: str, default 'mrsm' 144 The virtual environment to deactivate. 145 146 color: bool, default True 147 If `True`, include color in debug text. 148 149 debug: bool, default False 150 Verbosity toggle. 151 152 previously_active_venvs: Union[Set[str], List[str], None] 153 If provided, skip deactivating if a virtual environment is in this iterable. 154 155 force: bool, default False 156 If `True`, forcibly deactivate the virtual environment. 157 This may cause issues with other threads, so be careful! 158 159 Returns 160 ------- 161 Return a bool indicating whether the virtual environment was successfully deactivated. 162 163 """ 164 import sys 165 thread_id = get_ident() 166 if venv is None: 167 if venv in active_venvs: 168 active_venvs.remove(venv) 169 return True 170 171 if previously_active_venvs and venv in previously_active_venvs and not force: 172 return True 173 174 with LOCKS['active_venvs']: 175 if venv in threads_active_venvs.get(thread_id, {}): 176 new_count = threads_active_venvs[thread_id][venv] - 1 177 if new_count > 0 and not force: 178 threads_active_venvs[thread_id][venv] = new_count 179 return True 180 else: 181 del threads_active_venvs[thread_id][venv] 182 183 if not force: 184 for other_thread_id, other_venvs in threads_active_venvs.items(): 185 if other_thread_id == thread_id: 186 continue 187 if venv in other_venvs: 188 return True 189 else: 190 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 191 for other_thread_id in to_delete: 192 del threads_active_venvs[other_thread_id] 193 194 if venv in active_venvs: 195 active_venvs.remove(venv) 196 197 if sys.path is None: 198 return False 199 200 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 201 with LOCKS['sys.path']: 202 if target in sys.path: 203 try: 204 sys.path.remove(target) 205 except Exception: 206 pass 207 try: 208 active_venvs_order.remove(venv) 209 except Exception: 210 pass 211 212 return True
Remove a virtual environment from sys.path (if it's been activated).
Parameters
- venv (str, default 'mrsm'): The virtual environment to deactivate.
- color (bool, default True):
If
True, include color in debug text. - debug (bool, default False): Verbosity toggle.
- previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
- force (bool, default False):
If
True, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
- Return a bool indicating whether the virtual environment was successfully deactivated.
852def get_module_venv(module) -> Union[str, None]: 853 """ 854 Return the virtual environment where an imported module is installed. 855 856 Parameters 857 ---------- 858 module: ModuleType 859 The imported module to inspect. 860 861 Returns 862 ------- 863 The name of a venv or `None`. 864 """ 865 import meerschaum.config.paths as paths 866 module_path = pathlib.Path(module.__file__).resolve() 867 try: 868 rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH) 869 except ValueError: 870 return None 871 872 return rel_path.as_posix().split('/', maxsplit=1)[0]
Return the virtual environment where an imported module is installed.
Parameters
- module (ModuleType): The imported module to inspect.
Returns
- The name of a venv or
None.
835def get_venvs() -> List[str]: 836 """ 837 Return a list of all the virtual environments. 838 """ 839 import os 840 import meerschaum.config.paths as paths 841 venvs = [] 842 for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH): 843 path = paths.VIRTENV_RESOURCES_PATH / filename 844 if not path.is_dir(): 845 continue 846 if not venv_exists(filename): 847 continue 848 venvs.append(filename) 849 return venvs
Return a list of all the virtual environments.
371def init_venv( 372 venv: str = 'mrsm', 373 verify: bool = True, 374 force: bool = False, 375 debug: bool = False, 376) -> bool: 377 """ 378 Initialize the virtual environment. 379 380 Parameters 381 ---------- 382 venv: str, default 'mrsm' 383 The name of the virtual environment to create. 384 385 verify: bool, default True 386 If `True`, verify that the virtual environment is in the expected state. 387 388 force: bool, default False 389 If `True`, recreate the virtual environment, even if already initalized. 390 391 Returns 392 ------- 393 A `bool` indicating success. 394 """ 395 if not force and venv in verified_venvs: 396 return True 397 if not force and venv_exists(venv, debug=debug): 398 if verify: 399 verify_venv(venv, debug=debug) 400 verified_venvs.add(venv) 401 return True 402 403 import io 404 from contextlib import redirect_stdout 405 import sys 406 import platform 407 import os 408 import shutil 409 import time 410 411 import meerschaum.config.paths as paths 412 from meerschaum._internal.static import STATIC_CONFIG 413 from meerschaum.utils.packages import is_uv_enabled 414 415 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 416 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 417 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 418 lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 419 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 420 421 def update_lock(active: bool): 422 try: 423 if not active: 424 if debug: 425 print(f"Releasing lock: '{lock_path}'") 426 lock_path.unlink() 427 else: 428 if debug: 429 print(f"Acquiring lock: '{lock_path}'") 430 lock_path.touch() 431 except Exception: 432 pass 433 434 def wait_for_lock(): 435 if platform.system() == 'Windows': 436 return 437 max_lock_seconds = 30.0 438 sleep_message_seconds = 5.0 439 step_sleep_seconds = 0.1 440 init_venv_check_start = time.perf_counter() 441 last_print = init_venv_check_start 442 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 443 if not lock_path.exists(): 444 break 445 446 now = time.perf_counter() 447 if debug or (now - last_print) > sleep_message_seconds: 448 print(f"Lock exists for venv '{venv}', sleeping...") 449 last_print = now 450 time.sleep(step_sleep_seconds) 451 update_lock(False) 452 453 if ( 454 not force 455 and venv == 'mrsm' 456 and os.environ.get(work_dir_env_var, None) is not None 457 and docker_home_venv_path.exists() 458 ): 459 wait_for_lock() 460 shutil.move(docker_home_venv_path, venv_path) 461 if verify: 462 verify_venv(venv, debug=debug) 463 verified_venvs.add(venv) 464 return True 465 466 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 467 global tried_virtualenv 468 try: 469 import venv as _venv 470 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 471 virtualenv = None 472 except ImportError: 473 _venv = None 474 uv = None 475 virtualenv = None 476 477 _venv_success = False 478 temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv) 479 ### NOTE: Disable site-packages movement for now. 480 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 481 482 wait_for_lock() 483 update_lock(True) 484 485 if rename_vtp: 486 if debug: 487 print(f"Moving '{vtp}' to '{temp_vtp}'...") 488 shutil.move(vtp, temp_vtp) 489 490 if uv is not None: 491 _venv_success = run_python_package( 492 'uv', 493 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 494 venv=None, 495 env=_get_pip_os_env(), 496 debug=debug, 497 ) == 0 498 499 if _venv is not None and not _venv_success: 500 f = io.StringIO() 501 with redirect_stdout(f): 502 _venv_success = run_python_package( 503 'venv', 504 [venv_path.as_posix()] + ( 505 ['--symlinks'] 506 if platform.system() != 'Windows' 507 else [] 508 ), 509 venv=None, debug=debug 510 ) == 0 511 if not _venv_success: 512 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 513 if not venv_exists(venv, debug=debug): 514 _venv = None 515 if not _venv_success: 516 virtualenv = attempt_import( 517 'virtualenv', 518 venv=None, 519 lazy=False, 520 install=(not tried_virtualenv), 521 warn=False, 522 check_update=False, 523 color=False, 524 debug=debug, 525 ) 526 if virtualenv is None: 527 print( 528 "Failed to import `venv` or `virtualenv`! " 529 + "Please install `virtualenv` via pip then restart Meerschaum." 530 ) 531 if rename_vtp and temp_vtp.exists(): 532 if debug: 533 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 534 shutil.move(temp_vtp, vtp) 535 update_lock(False) 536 return False 537 538 tried_virtualenv = True 539 try: 540 python_folder = ( 541 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 542 ) 543 dist_packages_path = ( 544 paths.VIRTENV_RESOURCES_PATH / 545 venv / 'local' / 'lib' / python_folder / 'dist-packages' 546 ) 547 local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 548 bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin' 549 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 550 if bin_path.exists(): 551 try: 552 shutil.rmtree(bin_path) 553 except Exception: 554 import traceback 555 traceback.print_exc() 556 virtualenv.cli_run([venv_path.as_posix()]) 557 if dist_packages_path.exists(): 558 vtp.mkdir(exist_ok=True, parents=True) 559 for file_path in dist_packages_path.glob('*'): 560 shutil.move(file_path, vtp) 561 shutil.rmtree(dist_packages_path) 562 # shutil.move(dist_packages_path, vtp) 563 bin_path.mkdir(exist_ok=True, parents=True) 564 for file_path in local_bin_path.glob('*'): 565 shutil.move(file_path, bin_path) 566 # shutil.move(local_bin_path, bin_path) 567 shutil.rmtree(local_bin_path) 568 569 except Exception: 570 import traceback 571 traceback.print_exc() 572 if rename_vtp and temp_vtp.exists(): 573 shutil.move(temp_vtp, vtp) 574 update_lock(False) 575 return False 576 if verify: 577 verify_venv(venv, debug=debug) 578 verified_venvs.add(venv) 579 580 if rename_vtp and temp_vtp.exists(): 581 if debug: 582 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 583 shutil.move(temp_vtp, vtp) 584 585 update_lock(False) 586 return True
Initialize the virtual environment.
Parameters
- venv (str, default 'mrsm'): The name of the virtual environment to create.
- verify (bool, default True):
If
True, verify that the virtual environment is in the expected state. - force (bool, default False):
If
True, recreate the virtual environment, even if already initalized.
Returns
- A
boolindicating success.
822def inside_venv() -> bool: 823 """ 824 Determine whether current Python interpreter is running inside a virtual environment. 825 """ 826 import sys 827 return ( 828 hasattr(sys, 'real_prefix') or ( 829 hasattr(sys, 'base_prefix') 830 and sys.base_prefix != sys.prefix 831 ) 832 )
Determine whether current Python interpreter is running inside a virtual environment.
215def is_venv_active( 216 venv: str = 'mrsm', 217 color : bool = True, 218 debug: bool = False 219) -> bool: 220 """ 221 Check if a virtual environment is active. 222 223 Parameters 224 ---------- 225 venv: str, default 'mrsm' 226 The virtual environment to check. 227 228 color: bool, default True 229 If `True`, include color in debug text. 230 231 debug: bool, default False 232 Verbosity toggle. 233 234 Returns 235 ------- 236 A bool indicating whether the virtual environment `venv` is active. 237 238 """ 239 return venv in active_venvs
Check if a virtual environment is active.
Parameters
- venv (str, default 'mrsm'): The virtual environment to check.
- color (bool, default True):
If
True, include color in debug text. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment
venvis active.
614def venv_exec( 615 code: str, 616 venv: Optional[str] = 'mrsm', 617 env: Optional[Dict[str, str]] = None, 618 with_extras: bool = False, 619 as_proc: bool = False, 620 capture_output: bool = True, 621 debug: bool = False, 622) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 623 """ 624 Execute Python code in a subprocess via a virtual environment's interpeter. 625 Return `True` if the code successfully executes, `False` on failure. 626 627 Parameters 628 ---------- 629 code: str 630 The Python code to excecute. 631 632 venv: str, default 'mrsm' 633 The virtual environment to use to get the path for the Python executable. 634 If `venv` is `None`, use the default `sys.executable` path. 635 636 env: Optional[Dict[str, str]], default None 637 Optionally specify the environment variables for the subprocess. 638 Defaults to `os.environ`. 639 640 with_extras: bool, default False 641 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 642 643 as_proc: bool, default False 644 If `True`, return the `subprocess.Popen` object instead of executing. 645 646 Returns 647 ------- 648 By default, return a bool indicating success. 649 If `as_proc` is `True`, return a `subprocess.Popen` object. 650 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 651 652 """ 653 import os 654 import subprocess 655 import platform 656 from meerschaum.utils.debug import dprint 657 from meerschaum.utils.process import _child_processes 658 659 executable = venv_executable(venv=venv) 660 cmd_list = [executable, '-c', code] 661 if env is None: 662 env = os.environ 663 if debug: 664 dprint(str(cmd_list)) 665 if not with_extras and not as_proc: 666 return subprocess.call(cmd_list, env=env) == 0 667 668 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 669 group_kwargs = ( 670 { 671 'preexec_fn': os.setsid, 672 } if platform.system() != 'Windows' 673 else { 674 'creationflags': CREATE_NEW_PROCESS_GROUP, 675 } 676 ) 677 process = subprocess.Popen( 678 cmd_list, 679 stdout=stdout, 680 stderr=stderr, 681 stdin=sys.stdin, 682 env=env, 683 **group_kwargs 684 ) 685 if as_proc: 686 _child_processes.append(process) 687 return process 688 stdout, stderr = process.communicate() 689 exit_code = process.returncode 690 return exit_code, stdout, stderr
Execute Python code in a subprocess via a virtual environment's interpeter.
Return True if the code successfully executes, False on failure.
Parameters
- code (str): The Python code to excecute.
- venv (str, default 'mrsm'):
The virtual environment to use to get the path for the Python executable.
If
venvisNone, use the defaultsys.executablepath. - env (Optional[Dict[str, str]], default None):
Optionally specify the environment variables for the subprocess.
Defaults to
os.environ. - with_extras (bool, default False):
If
True, return a tuple of the exit code, stdout bytes, and stderr bytes. - as_proc (bool, default False):
If
True, return thesubprocess.Popenobject instead of executing.
Returns
- By default, return a bool indicating success.
- If
as_procisTrue, return asubprocess.Popenobject. - If
with_extrasisTrue, return a tuple of the exit code, stdout bytes, and stderr bytes.
589def venv_executable(venv: Optional[str] = 'mrsm') -> str: 590 """ 591 The Python interpreter executable for a given virtual environment. 592 """ 593 import sys 594 import platform 595 import meerschaum.config.paths as paths 596 return ( 597 sys.executable if venv is None 598 else str( 599 paths.VIRTENV_RESOURCES_PATH 600 / venv 601 / ( 602 'bin' if platform.system() != 'Windows' 603 else 'Scripts' 604 ) / ( 605 'python' 606 + str(sys.version_info.major) 607 + '.' 608 + str(sys.version_info.minor) 609 ) 610 ) 611 )
The Python interpreter executable for a given virtual environment.
693def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 694 """ 695 Determine whether a virtual environment has been created. 696 """ 697 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 698 return target_path.exists()
Determine whether a virtual environment has been created.
701def venv_target_path( 702 venv: Union[str, None], 703 allow_nonexistent: bool = False, 704 debug: bool = False, 705) -> pathlib.Path: 706 """ 707 Return a virtual environment's site-package path. 708 709 Parameters 710 ---------- 711 venv: Union[str, None] 712 The virtual environment for which a path should be returned. 713 714 allow_nonexistent: bool, default False 715 If `True`, return a path even if it does not exist. 716 717 Returns 718 ------- 719 The `pathlib.Path` object for the virtual environment's path. 720 721 """ 722 import os 723 import sys 724 import platform 725 import site 726 727 import meerschaum.config.paths as paths 728 from meerschaum._internal.static import STATIC_CONFIG 729 730 ### Check sys.path for a user-writable site-packages directory. 731 if venv is None: 732 733 ### Return the known value for the portable environment. 734 environment_runtime = STATIC_CONFIG['environment']['runtime'] 735 if os.environ.get(environment_runtime, None) == 'portable': 736 python_version_folder = ( 737 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 738 ) 739 executable_path = pathlib.Path(sys.executable) 740 site_packages_path = ( 741 ( 742 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 743 ) if platform.system() != 'Windows' else ( 744 executable_path.parent / 'Lib' / 'site-packages' 745 ) 746 ) 747 if not site_packages_path.exists(): 748 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 749 return site_packages_path 750 751 if not inside_venv(): 752 user_site_packages = site.getusersitepackages() 753 if user_site_packages is None: 754 raise EnvironmentError("Could not determine user site packages.") 755 756 site_path = pathlib.Path(user_site_packages) 757 if not site_path.exists(): 758 759 ### Windows does not have `os.geteuid()`. 760 if platform.system() == 'Windows' or os.geteuid() != 0: 761 site_path.mkdir(parents=True, exist_ok=True) 762 return site_path 763 764 ### Allow for dist-level paths (running as root). 765 for possible_dist in site.getsitepackages(): 766 dist_path = pathlib.Path(possible_dist) 767 if not dist_path.exists(): 768 continue 769 return dist_path 770 771 raise EnvironmentError("Could not determine the dist-packages directory.") 772 773 return site_path 774 775 venv_root_path = ( 776 (paths.VIRTENV_RESOURCES_PATH / venv) 777 if venv is not None 778 else pathlib.Path(sys.prefix) 779 ) 780 target_path = venv_root_path 781 782 ### Ensure 'lib' or 'Lib' exists. 783 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 784 if not allow_nonexistent: 785 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 786 print(f"Failed to find lib directory for virtual environment '{venv}'.") 787 import traceback 788 traceback.print_stack() 789 sys.exit(1) 790 target_path = target_path / lib 791 792 ### Check if a 'python3.x' folder exists. 793 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 794 if target_path.exists(): 795 target_path = ( 796 (target_path / python_folder) 797 if python_folder in os.listdir(target_path) 798 else target_path 799 ) 800 else: 801 target_path = ( 802 (target_path / python_folder) 803 if platform.system() != 'Windows' 804 else target_path 805 ) 806 807 ### Ensure 'site-packages' exists. 808 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 809 target_path = target_path / 'site-packages' 810 else: 811 import traceback 812 traceback.print_stack() 813 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 814 print("This may be because you are using a different Python version.") 815 print("Try deleting the following directory and restarting Meerschaum:") 816 print(paths.VIRTENV_RESOURCES_PATH) 817 sys.exit(1) 818 819 return target_path
Return a virtual environment's site-package path.
Parameters
- venv (Union[str, None]): The virtual environment for which a path should be returned.
- allow_nonexistent (bool, default False):
If
True, return a path even if it does not exist.
Returns
- The
pathlib.Pathobject for the virtual environment's path.
243def verify_venv( 244 venv: str, 245 debug: bool = False, 246) -> None: 247 """ 248 Verify that the virtual environment matches the expected state. 249 """ 250 import platform 251 import os 252 import shutil 253 import sys 254 255 import meerschaum.config.paths as paths 256 from meerschaum.utils.process import run_process 257 from meerschaum.utils.misc import make_symlink, is_symlink 258 from meerschaum.utils.warnings import warn 259 260 venv_path = paths.VIRTENV_RESOURCES_PATH / venv 261 bin_path = venv_path / ( 262 'bin' if platform.system() != 'Windows' else "Scripts" 263 ) 264 current_python_versioned_name = ( 265 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 266 + ('' if platform.system() != 'Windows' else '.exe') 267 ) 268 269 if not (bin_path / current_python_versioned_name).exists(): 270 init_venv(venv, verify=False, force=True, debug=debug) 271 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 272 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 273 if not current_python_in_venv_path.exists(): 274 if is_symlink(current_python_in_venv_path): 275 try: 276 current_python_in_venv_path.unlink() 277 except Exception as e: 278 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 279 try: 280 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 281 except Exception: 282 print( 283 f"Unable to create symlink {current_python_in_venv_path} " 284 + f"to {current_python_in_sys_path}." 285 ) 286 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 287 else: 288 files_to_inspect = [current_python_versioned_name] 289 290 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 291 """ 292 Return the version for the python binary at the given path. 293 """ 294 try: 295 ### It might be a broken symlink, so skip on errors. 296 if debug: 297 print(f"Getting python version for {python_path}") 298 proc = run_process( 299 [str(python_path), '-V'], 300 as_proc=True, 301 capture_output=True, 302 ) 303 stdout, stderr = proc.communicate(timeout=1.0) 304 except Exception as e: 305 ### E.g. the symlink may be broken. 306 if is_symlink(python_path): 307 try: 308 python_path.unlink() 309 except Exception as _e: 310 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 311 return None 312 return stdout.decode('utf-8').strip().replace('Python ', '') 313 314 ### Ensure the versions are symlinked correctly. 315 for filename in files_to_inspect: 316 if not filename.startswith('python'): 317 continue 318 python_path = bin_path / filename 319 version = get_python_version(python_path) 320 if version is None: 321 continue 322 try: 323 major_version = version.split('.', maxsplit=1)[0] 324 minor_version = version.split('.', maxsplit=2)[1] 325 except IndexError: 326 return 327 python_versioned_name = ( 328 'python' + major_version + '.' + minor_version 329 + ('' if platform.system() != 'Windows' else '.exe') 330 ) 331 332 ### E.g. python3.10 actually links to Python 3.10. 333 if filename == python_versioned_name: 334 try: 335 real_path = pathlib.Path(os.path.realpath(python_path)) 336 real_path_exists = real_path.exists() 337 except Exception: 338 real_path_exists = False 339 340 if not real_path_exists: 341 try: 342 python_path.unlink() 343 except Exception: 344 pass 345 init_venv(venv, verify=False, force=True, debug=debug) 346 if not python_path.exists(): 347 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 348 349 if python_path == real_path: 350 continue 351 352 try: 353 python_path.unlink() 354 except Exception: 355 pass 356 success, msg = make_symlink(real_path, python_path) 357 if not success: 358 warn(msg, color=False) 359 continue 360 361 python_versioned_path = bin_path / python_versioned_name 362 if python_versioned_path.exists(): 363 ### Avoid circular symlinks. 364 if get_python_version(python_versioned_path) == version: 365 continue 366 python_versioned_path.unlink() 367 shutil.move(python_path, python_versioned_path)
Verify that the virtual environment matches the expected state.