meerschaum.utils.venv
Manage virtual environments.
1#! /usr/bin/env python3 2# -*- coding: utf-8 -*- 3# vim:fenc=utf-8 4 5""" 6Manage virtual environments. 7""" 8 9from __future__ import annotations 10 11from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple 12from meerschaum.utils.threading import RLock, get_ident 13 14__all__ = sorted([ 15 'activate_venv', 'deactivate_venv', 'init_venv', 16 'inside_venv', 'is_venv_active', 'venv_exec', 17 'venv_executable', 'venv_exists', 'venv_target_path', 18 'Venv', 'get_venvs', 'verify_venv', 'get_module_venv', 19]) 20__pdoc__ = {'Venv': True} 21 22LOCKS = { 23 'sys.path': RLock(), 24 'active_venvs': RLock(), 25 'venvs_active_counts': RLock(), 26} 27 28active_venvs = set() 29active_venvs_counts: Dict[str, int] = {} 30active_venvs_order: List[Optional[str]] = [] 31threads_active_venvs: Dict[int, 'set[str]'] = {} 32CREATE_NEW_PROCESS_GROUP = 0x00000200 33 34 35def activate_venv( 36 venv: Optional[str] = 'mrsm', 37 color: bool = True, 38 force: bool = False, 39 debug: bool = False, 40 **kw 41) -> bool: 42 """ 43 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 44 45 Parameters 46 ---------- 47 venv: Optional[str], default 'mrsm' 48 The virtual environment to activate. 49 50 color: bool, default True 51 If `True`, include color in debug text. 52 53 force: bool, default False 54 If `True`, do not exit early even if the venv is currently active. 55 56 debug: bool, default False 57 Verbosity toggle. 58 59 Returns 60 ------- 61 A bool indicating whether the virtual environment was successfully activated. 62 63 """ 64 thread_id = get_ident() 65 if active_venvs_order and active_venvs_order[0] == venv: 66 if not force: 67 return True 68 import sys 69 import os 70 if venv is not None: 71 init_venv(venv=venv, debug=debug) 72 with LOCKS['active_venvs']: 73 if thread_id not in threads_active_venvs: 74 threads_active_venvs[thread_id] = {} 75 active_venvs.add(venv) 76 if venv not in threads_active_venvs[thread_id]: 77 threads_active_venvs[thread_id][venv] = 1 78 else: 79 threads_active_venvs[thread_id][venv] += 1 80 81 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 82 if not target_path.exists(): 83 init_venv(venv=venv, force=True, debug=debug) 84 if not target_path.exists(): 85 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 86 target = target_path.as_posix() 87 88 if venv in active_venvs_order: 89 sys.path.remove(target) 90 try: 91 active_venvs_order.remove(venv) 92 except Exception: 93 pass 94 if venv is not None: 95 sys.path.insert(0, target) 96 else: 97 if sys.path and sys.path[0] in (os.getcwd(), ''): 98 sys.path.insert(1, target) 99 else: 100 sys.path.insert(0, target) 101 try: 102 active_venvs_order.insert(0, venv) 103 except Exception: 104 pass 105 106 return True 107 108 109def deactivate_venv( 110 venv: str = 'mrsm', 111 color: bool = True, 112 debug: bool = False, 113 previously_active_venvs: Union['set[str]', List[str], None] = None, 114 force: bool = False, 115 **kw 116) -> bool: 117 """ 118 Remove a virtual environment from `sys.path` (if it's been activated). 119 120 Parameters 121 ---------- 122 venv: str, default 'mrsm' 123 The virtual environment to deactivate. 124 125 color: bool, default True 126 If `True`, include color in debug text. 127 128 debug: bool, default False 129 Verbosity toggle. 130 131 previously_active_venvs: Union[Set[str], List[str], None] 132 If provided, skip deactivating if a virtual environment is in this iterable. 133 134 force: bool, default False 135 If `True`, forcibly deactivate the virtual environment. 136 This may cause issues with other threads, so be careful! 137 138 Returns 139 ------- 140 Return a bool indicating whether the virtual environment was successfully deactivated. 141 142 """ 143 import sys 144 thread_id = get_ident() 145 if venv is None: 146 if venv in active_venvs: 147 active_venvs.remove(venv) 148 return True 149 150 if previously_active_venvs and venv in previously_active_venvs and not force: 151 return True 152 153 with LOCKS['active_venvs']: 154 if venv in threads_active_venvs.get(thread_id, {}): 155 new_count = threads_active_venvs[thread_id][venv] - 1 156 if new_count > 0 and not force: 157 threads_active_venvs[thread_id][venv] = new_count 158 return True 159 else: 160 del threads_active_venvs[thread_id][venv] 161 162 if not force: 163 for other_thread_id, other_venvs in threads_active_venvs.items(): 164 if other_thread_id == thread_id: 165 continue 166 if venv in other_venvs: 167 return True 168 else: 169 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 170 for other_thread_id in to_delete: 171 del threads_active_venvs[other_thread_id] 172 173 if venv in active_venvs: 174 active_venvs.remove(venv) 175 176 if sys.path is None: 177 return False 178 179 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 180 with LOCKS['sys.path']: 181 if target in sys.path: 182 try: 183 sys.path.remove(target) 184 except Exception: 185 pass 186 try: 187 active_venvs_order.remove(venv) 188 except Exception: 189 pass 190 191 return True 192 193 194def is_venv_active( 195 venv: str = 'mrsm', 196 color : bool = True, 197 debug: bool = False 198) -> bool: 199 """ 200 Check if a virtual environment is active. 201 202 Parameters 203 ---------- 204 venv: str, default 'mrsm' 205 The virtual environment to check. 206 207 color: bool, default True 208 If `True`, include color in debug text. 209 210 debug: bool, default False 211 Verbosity toggle. 212 213 Returns 214 ------- 215 A bool indicating whether the virtual environment `venv` is active. 216 217 """ 218 return venv in active_venvs 219 220 221verified_venvs = set() 222def verify_venv( 223 venv: str, 224 debug: bool = False, 225) -> None: 226 """ 227 Verify that the virtual environment matches the expected state. 228 """ 229 import pathlib 230 import platform 231 import os 232 import shutil 233 import sys 234 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 235 from meerschaum.utils.process import run_process 236 from meerschaum.utils.misc import make_symlink, is_symlink 237 from meerschaum.utils.warnings import warn 238 239 venv_path = VIRTENV_RESOURCES_PATH / venv 240 bin_path = venv_path / ( 241 'bin' if platform.system() != 'Windows' else "Scripts" 242 ) 243 current_python_versioned_name = ( 244 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 245 + ('' if platform.system() != 'Windows' else '.exe') 246 ) 247 248 if not (bin_path / current_python_versioned_name).exists(): 249 init_venv(venv, verify=False, force=True, debug=debug) 250 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 251 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 252 if not current_python_in_venv_path.exists(): 253 if is_symlink(current_python_in_venv_path): 254 try: 255 current_python_in_venv_path.unlink() 256 except Exception as e: 257 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 258 try: 259 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 260 except Exception: 261 print( 262 f"Unable to create symlink {current_python_in_venv_path} " 263 + f"to {current_python_in_sys_path}." 264 ) 265 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 266 else: 267 files_to_inspect = [current_python_versioned_name] 268 269 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 270 """ 271 Return the version for the python binary at the given path. 272 """ 273 try: 274 ### It might be a broken symlink, so skip on errors. 275 if debug: 276 print(f"Getting python version for {python_path}") 277 proc = run_process( 278 [str(python_path), '-V'], 279 as_proc=True, 280 capture_output=True, 281 ) 282 stdout, stderr = proc.communicate(timeout=1.0) 283 except Exception as e: 284 ### E.g. the symlink may be broken. 285 if is_symlink(python_path): 286 try: 287 python_path.unlink() 288 except Exception as _e: 289 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 290 return None 291 return stdout.decode('utf-8').strip().replace('Python ', '') 292 293 ### Ensure the versions are symlinked correctly. 294 for filename in files_to_inspect: 295 if not filename.startswith('python'): 296 continue 297 python_path = bin_path / filename 298 version = get_python_version(python_path) 299 if version is None: 300 continue 301 try: 302 major_version = version.split('.', maxsplit=1)[0] 303 minor_version = version.split('.', maxsplit=2)[1] 304 except IndexError: 305 return 306 python_versioned_name = ( 307 'python' + major_version + '.' + minor_version 308 + ('' if platform.system() != 'Windows' else '.exe') 309 ) 310 311 ### E.g. python3.10 actually links to Python 3.10. 312 if filename == python_versioned_name: 313 real_path = pathlib.Path(os.path.realpath(python_path)) 314 if not real_path.exists(): 315 try: 316 python_path.unlink() 317 except Exception: 318 pass 319 init_venv(venv, verify=False, force=True, debug=debug) 320 if not python_path.exists(): 321 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 322 323 if python_path == real_path: 324 continue 325 326 try: 327 python_path.unlink() 328 except Exception: 329 pass 330 success, msg = make_symlink(real_path, python_path) 331 if not success: 332 warn(msg, color=False) 333 continue 334 335 python_versioned_path = bin_path / python_versioned_name 336 if python_versioned_path.exists(): 337 ### Avoid circular symlinks. 338 if get_python_version(python_versioned_path) == version: 339 continue 340 python_versioned_path.unlink() 341 shutil.move(python_path, python_versioned_path) 342 343 344tried_virtualenv = False 345def init_venv( 346 venv: str = 'mrsm', 347 verify: bool = True, 348 force: bool = False, 349 debug: bool = False, 350) -> bool: 351 """ 352 Initialize the virtual environment. 353 354 Parameters 355 ---------- 356 venv: str, default 'mrsm' 357 The name of the virtual environment to create. 358 359 verify: bool, default True 360 If `True`, verify that the virtual environment is in the expected state. 361 362 force: bool, default False 363 If `True`, recreate the virtual environment, even if already initalized. 364 365 Returns 366 ------- 367 A `bool` indicating success. 368 """ 369 if not force and venv in verified_venvs: 370 return True 371 if not force and venv_exists(venv, debug=debug): 372 if verify: 373 verify_venv(venv, debug=debug) 374 verified_venvs.add(venv) 375 return True 376 377 import io 378 from contextlib import redirect_stdout 379 import sys 380 import platform 381 import os 382 import pathlib 383 import shutil 384 import time 385 386 from meerschaum.config.static import STATIC_CONFIG 387 from meerschaum.config._paths import ( 388 VIRTENV_RESOURCES_PATH, 389 VENVS_CACHE_RESOURCES_PATH, 390 ) 391 from meerschaum.utils.packages import is_uv_enabled 392 393 venv_path = VIRTENV_RESOURCES_PATH / venv 394 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 395 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 396 lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 397 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 398 399 def update_lock(active: bool): 400 try: 401 if not active: 402 if debug: 403 print(f"Releasing lock: '{lock_path}'") 404 lock_path.unlink() 405 else: 406 if debug: 407 print(f"Acquiring lock: '{lock_path}'") 408 lock_path.touch() 409 except Exception: 410 pass 411 412 def wait_for_lock(): 413 max_lock_seconds = 30.0 414 sleep_message_seconds = 5.0 415 step_sleep_seconds = 0.1 416 init_venv_check_start = time.perf_counter() 417 last_print = init_venv_check_start 418 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 419 if not lock_path.exists(): 420 break 421 422 now = time.perf_counter() 423 if debug or (now - last_print) > sleep_message_seconds: 424 print(f"Lock exists for venv '{venv}', sleeping...") 425 last_print = now 426 time.sleep(step_sleep_seconds) 427 update_lock(False) 428 429 if ( 430 not force 431 and venv == 'mrsm' 432 and os.environ.get(work_dir_env_var, None) is not None 433 and docker_home_venv_path.exists() 434 ): 435 wait_for_lock() 436 shutil.move(docker_home_venv_path, venv_path) 437 if verify: 438 verify_venv(venv, debug=debug) 439 verified_venvs.add(venv) 440 return True 441 442 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 443 global tried_virtualenv 444 try: 445 import venv as _venv 446 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 447 virtualenv = None 448 except ImportError: 449 _venv = None 450 uv = None 451 virtualenv = None 452 453 _venv_success = False 454 temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv) 455 ### NOTE: Disable site-packages movement for now. 456 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 457 458 wait_for_lock() 459 update_lock(True) 460 461 if rename_vtp: 462 if debug: 463 print(f"Moving '{vtp}' to '{temp_vtp}'...") 464 shutil.move(vtp, temp_vtp) 465 466 if uv is not None: 467 _venv_success = run_python_package( 468 'uv', 469 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 470 venv=None, 471 env=_get_pip_os_env(), 472 debug=debug, 473 ) == 0 474 475 if _venv is not None and not _venv_success: 476 f = io.StringIO() 477 with redirect_stdout(f): 478 _venv_success = run_python_package( 479 'venv', 480 [venv_path.as_posix()] + ( 481 ['--symlinks'] 482 if platform.system() != 'Windows' 483 else [] 484 ), 485 venv=None, debug=debug 486 ) == 0 487 if not _venv_success: 488 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 489 if not venv_exists(venv, debug=debug): 490 _venv = None 491 if not _venv_success: 492 virtualenv = attempt_import( 493 'virtualenv', 494 venv=None, 495 lazy=False, 496 install=(not tried_virtualenv), 497 warn=False, 498 check_update=False, 499 color=False, 500 debug=debug, 501 ) 502 if virtualenv is None: 503 print( 504 "Failed to import `venv` or `virtualenv`! " 505 + "Please install `virtualenv` via pip then restart Meerschaum." 506 ) 507 if rename_vtp and temp_vtp.exists(): 508 if debug: 509 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 510 shutil.move(temp_vtp, vtp) 511 update_lock(False) 512 return False 513 514 tried_virtualenv = True 515 try: 516 python_folder = ( 517 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 518 ) 519 dist_packages_path = ( 520 VIRTENV_RESOURCES_PATH / 521 venv / 'local' / 'lib' / python_folder / 'dist-packages' 522 ) 523 local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 524 bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin' 525 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 526 if bin_path.exists(): 527 try: 528 shutil.rmtree(bin_path) 529 except Exception: 530 import traceback 531 traceback.print_exc() 532 virtualenv.cli_run([venv_path.as_posix()]) 533 if dist_packages_path.exists(): 534 vtp.mkdir(exist_ok=True, parents=True) 535 for file_path in dist_packages_path.glob('*'): 536 shutil.move(file_path, vtp) 537 shutil.rmtree(dist_packages_path) 538 # shutil.move(dist_packages_path, vtp) 539 bin_path.mkdir(exist_ok=True, parents=True) 540 for file_path in local_bin_path.glob('*'): 541 shutil.move(file_path, bin_path) 542 # shutil.move(local_bin_path, bin_path) 543 shutil.rmtree(local_bin_path) 544 545 except Exception: 546 import traceback 547 traceback.print_exc() 548 if rename_vtp and temp_vtp.exists(): 549 shutil.move(temp_vtp, vtp) 550 update_lock(False) 551 return False 552 if verify: 553 verify_venv(venv, debug=debug) 554 verified_venvs.add(venv) 555 556 if rename_vtp and temp_vtp.exists(): 557 if debug: 558 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 559 shutil.move(temp_vtp, vtp) 560 561 update_lock(False) 562 return True 563 564 565def venv_executable(venv: Optional[str] = 'mrsm') -> str: 566 """ 567 The Python interpreter executable for a given virtual environment. 568 """ 569 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 570 import sys 571 import platform 572 return ( 573 sys.executable if venv is None 574 else str( 575 VIRTENV_RESOURCES_PATH 576 / venv 577 / ( 578 'bin' if platform.system() != 'Windows' 579 else 'Scripts' 580 ) / ( 581 'python' 582 + str(sys.version_info.major) 583 + '.' 584 + str(sys.version_info.minor) 585 ) 586 ) 587 ) 588 589 590def venv_exec( 591 code: str, 592 venv: Optional[str] = 'mrsm', 593 env: Optional[Dict[str, str]] = None, 594 with_extras: bool = False, 595 as_proc: bool = False, 596 capture_output: bool = True, 597 debug: bool = False, 598) -> Union[bool, Tuple[int, bytes, bytes]]: 599 """ 600 Execute Python code in a subprocess via a virtual environment's interpeter. 601 Return `True` if the code successfully executes, `False` on failure. 602 603 Parameters 604 ---------- 605 code: str 606 The Python code to excecute. 607 608 venv: str, default 'mrsm' 609 The virtual environment to use to get the path for the Python executable. 610 If `venv` is `None`, use the default `sys.executable` path. 611 612 env: Optional[Dict[str, str]], default None 613 Optionally specify the environment variables for the subprocess. 614 Defaults to `os.environ`. 615 616 with_extras: bool, default False 617 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 618 619 as_proc: bool, default False 620 If `True`, return the `subprocess.Popen` object instead of executing. 621 622 Returns 623 ------- 624 By default, return a bool indicating success. 625 If `as_proc` is `True`, return a `subprocess.Popen` object. 626 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 627 628 """ 629 import os 630 import subprocess 631 import platform 632 from meerschaum.utils.debug import dprint 633 executable = venv_executable(venv=venv) 634 cmd_list = [executable, '-c', code] 635 if env is None: 636 env = os.environ 637 if debug: 638 dprint(str(cmd_list)) 639 if not with_extras and not as_proc: 640 return subprocess.call(cmd_list, env=env) == 0 641 642 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 643 group_kwargs = ( 644 { 645 'preexec_fn': os.setsid, 646 } if platform.system() != 'Windows' 647 else { 648 'creationflags': CREATE_NEW_PROCESS_GROUP, 649 } 650 ) 651 process = subprocess.Popen( 652 cmd_list, 653 stdout=stdout, 654 stderr=stderr, 655 env=env, 656 **group_kwargs 657 ) 658 if as_proc: 659 return process 660 stdout, stderr = process.communicate() 661 exit_code = process.returncode 662 return exit_code, stdout, stderr 663 664 665def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 666 """ 667 Determine whether a virtual environment has been created. 668 """ 669 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 670 return target_path.exists() 671 672 673def venv_target_path( 674 venv: Union[str, None], 675 allow_nonexistent: bool = False, 676 debug: bool = False, 677) -> 'pathlib.Path': 678 """ 679 Return a virtual environment's site-package path. 680 681 Parameters 682 ---------- 683 venv: Union[str, None] 684 The virtual environment for which a path should be returned. 685 686 allow_nonexistent: bool, default False 687 If `True`, return a path even if it does not exist. 688 689 Returns 690 ------- 691 The `pathlib.Path` object for the virtual environment's path. 692 693 """ 694 import os 695 import sys 696 import platform 697 import pathlib 698 import site 699 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 700 from meerschaum.config.static import STATIC_CONFIG 701 702 ### Check sys.path for a user-writable site-packages directory. 703 if venv is None: 704 705 ### Return the known value for the portable environment. 706 environment_runtime = STATIC_CONFIG['environment']['runtime'] 707 if os.environ.get(environment_runtime, None) == 'portable': 708 python_version_folder = ( 709 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 710 ) 711 executable_path = pathlib.Path(sys.executable) 712 site_packages_path = ( 713 ( 714 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 715 ) if platform.system() != 'Windows' else ( 716 executable_path.parent / 'Lib' / 'site-packages' 717 ) 718 ) 719 if not site_packages_path.exists(): 720 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 721 return site_packages_path 722 723 if not inside_venv(): 724 user_site_packages = site.getusersitepackages() 725 if user_site_packages is None: 726 raise EnvironmentError("Could not determine user site packages.") 727 728 site_path = pathlib.Path(user_site_packages) 729 if not site_path.exists(): 730 731 ### Windows does not have `os.geteuid()`. 732 if platform.system() == 'Windows' or os.geteuid() != 0: 733 site_path.mkdir(parents=True, exist_ok=True) 734 return site_path 735 736 ### Allow for dist-level paths (running as root). 737 for possible_dist in site.getsitepackages(): 738 dist_path = pathlib.Path(possible_dist) 739 if not dist_path.exists(): 740 continue 741 return dist_path 742 743 raise EnvironmentError("Could not determine the dist-packages directory.") 744 745 return site_path 746 747 venv_root_path = ( 748 (VIRTENV_RESOURCES_PATH / venv) 749 if venv is not None else pathlib.Path(sys.prefix) 750 ) 751 target_path = venv_root_path 752 753 ### Ensure 'lib' or 'Lib' exists. 754 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 755 if not allow_nonexistent: 756 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 757 print(f"Failed to find lib directory for virtual environment '{venv}'.") 758 import traceback 759 traceback.print_stack() 760 sys.exit(1) 761 target_path = target_path / lib 762 763 ### Check if a 'python3.x' folder exists. 764 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 765 if target_path.exists(): 766 target_path = ( 767 (target_path / python_folder) if python_folder in os.listdir(target_path) 768 else target_path 769 ) 770 else: 771 target_path = ( 772 (target_path / python_folder) if platform.system() != 'Windows' 773 else target_path 774 ) 775 776 ### Ensure 'site-packages' exists. 777 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 778 target_path = target_path / 'site-packages' 779 else: 780 import traceback 781 traceback.print_stack() 782 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 783 print("This may be because you are using a different Python version.") 784 print("Try deleting the following directory and restarting Meerschaum:") 785 print(VIRTENV_RESOURCES_PATH) 786 sys.exit(1) 787 788 return target_path 789 790 791def inside_venv() -> bool: 792 """ 793 Determine whether current Python interpreter is running inside a virtual environment. 794 """ 795 import sys 796 return ( 797 hasattr(sys, 'real_prefix') or ( 798 hasattr(sys, 'base_prefix') 799 and sys.base_prefix != sys.prefix 800 ) 801 ) 802 803 804def get_venvs() -> List[str]: 805 """ 806 Return a list of all the virtual environments. 807 """ 808 import os 809 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 810 venvs = [] 811 for filename in os.listdir(VIRTENV_RESOURCES_PATH): 812 path = VIRTENV_RESOURCES_PATH / filename 813 if not path.is_dir(): 814 continue 815 if not venv_exists(filename): 816 continue 817 venvs.append(filename) 818 return venvs 819 820 821def get_module_venv(module) -> Union[str, None]: 822 """ 823 Return the virtual environment where an imported module is installed. 824 825 Parameters 826 ---------- 827 module: ModuleType 828 The imported module to inspect. 829 830 Returns 831 ------- 832 The name of a venv or `None`. 833 """ 834 import pathlib 835 from meerschaum.config.paths import VIRTENV_RESOURCES_PATH 836 module_path = pathlib.Path(module.__file__).resolve() 837 try: 838 rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH) 839 except ValueError: 840 return None 841 842 return rel_path.as_posix().split('/', maxsplit=1)[0] 843 844 845from meerschaum.utils.venv._Venv import Venv
18class Venv: 19 """ 20 Manage a virtual enviroment's activation status. 21 22 Examples 23 -------- 24 >>> from meerschaum.plugins import Plugin 25 >>> with Venv('mrsm') as venv: 26 ... import pandas 27 >>> with Venv(Plugin('noaa')) as venv: 28 ... import requests 29 >>> venv = Venv('mrsm') 30 >>> venv.activate() 31 True 32 >>> venv.deactivate() 33 True 34 >>> 35 """ 36 37 def __init__( 38 self, 39 venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm', 40 debug: bool = False, 41 ) -> None: 42 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 43 ### For some weird threading issue, 44 ### we can't use `isinstance` here. 45 if 'meerschaum.plugins._Plugin' in str(type(venv)): 46 self._venv = venv.name 47 self._activate = venv.activate_venv 48 self._deactivate = venv.deactivate_venv 49 self._kwargs = {} 50 else: 51 self._venv = venv 52 self._activate = activate_venv 53 self._deactivate = deactivate_venv 54 self._kwargs = {'venv': venv} 55 self._debug = debug 56 ### In case someone calls `deactivate()` before `activate()`. 57 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 58 59 60 def activate(self, debug: bool = False) -> bool: 61 """ 62 Activate this virtual environment. 63 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 64 will also be activated. 65 """ 66 from meerschaum.utils.venv import active_venvs, init_venv 67 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 68 try: 69 return self._activate(debug=(debug or self._debug), **self._kwargs) 70 except OSError as e: 71 if not init_venv(self._venv, force=True): 72 raise e 73 return self._activate(debug=(debug or self._debug), **self._kwargs) 74 75 76 def deactivate(self, debug: bool = False) -> bool: 77 """ 78 Deactivate this virtual environment. 79 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 80 will also be deactivated. 81 """ 82 return self._deactivate(debug=(debug or self._debug), **self._kwargs) 83 84 85 @property 86 def target_path(self) -> pathlib.Path: 87 """ 88 Return the target site-packages path for this virtual environment. 89 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 90 (e.g. Python 3.10 and Python 3.7). 91 """ 92 from meerschaum.utils.venv import venv_target_path 93 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug) 94 95 96 @property 97 def root_path(self) -> pathlib.Path: 98 """ 99 Return the top-level path for this virtual environment. 100 """ 101 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 102 if self._venv is None: 103 return self.target_path.parent 104 return VIRTENV_RESOURCES_PATH / self._venv 105 106 107 def __enter__(self) -> None: 108 self.activate(debug=self._debug) 109 110 111 def __exit__(self, exc_type, exc_value, exc_traceback) -> None: 112 self.deactivate(debug=self._debug) 113 114 115 def __str__(self) -> str: 116 quote = "'" if self._venv is not None else "" 117 return "Venv(" + quote + str(self._venv) + quote + ")" 118 119 120 def __repr__(self) -> str: 121 return self.__str__()
Manage a virtual enviroment's activation status.
Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
... import pandas
>>> with Venv(Plugin('noaa')) as venv:
... import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
37 def __init__( 38 self, 39 venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm', 40 debug: bool = False, 41 ) -> None: 42 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 43 ### For some weird threading issue, 44 ### we can't use `isinstance` here. 45 if 'meerschaum.plugins._Plugin' in str(type(venv)): 46 self._venv = venv.name 47 self._activate = venv.activate_venv 48 self._deactivate = venv.deactivate_venv 49 self._kwargs = {} 50 else: 51 self._venv = venv 52 self._activate = activate_venv 53 self._deactivate = deactivate_venv 54 self._kwargs = {'venv': venv} 55 self._debug = debug 56 ### In case someone calls `deactivate()` before `activate()`. 57 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
60 def activate(self, debug: bool = False) -> bool: 61 """ 62 Activate this virtual environment. 63 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 64 will also be activated. 65 """ 66 from meerschaum.utils.venv import active_venvs, init_venv 67 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 68 try: 69 return self._activate(debug=(debug or self._debug), **self._kwargs) 70 except OSError as e: 71 if not init_venv(self._venv, force=True): 72 raise e 73 return self._activate(debug=(debug or self._debug), **self._kwargs)
Activate this virtual environment.
If a meerschaum.plugins.Plugin
was provided, its dependent virtual environments
will also be activated.
76 def deactivate(self, debug: bool = False) -> bool: 77 """ 78 Deactivate this virtual environment. 79 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 80 will also be deactivated. 81 """ 82 return self._deactivate(debug=(debug or self._debug), **self._kwargs)
Deactivate this virtual environment.
If a meerschaum.plugins.Plugin
was provided, its dependent virtual environments
will also be deactivated.
85 @property 86 def target_path(self) -> pathlib.Path: 87 """ 88 Return the target site-packages path for this virtual environment. 89 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 90 (e.g. Python 3.10 and Python 3.7). 91 """ 92 from meerschaum.utils.venv import venv_target_path 93 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
Return the target site-packages path for this virtual environment.
A meerschaum.utils.venv.Venv
may have one virtual environment per minor Python version
(e.g. Python 3.10 and Python 3.7).
96 @property 97 def root_path(self) -> pathlib.Path: 98 """ 99 Return the top-level path for this virtual environment. 100 """ 101 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 102 if self._venv is None: 103 return self.target_path.parent 104 return VIRTENV_RESOURCES_PATH / self._venv
Return the top-level path for this virtual environment.
36def activate_venv( 37 venv: Optional[str] = 'mrsm', 38 color: bool = True, 39 force: bool = False, 40 debug: bool = False, 41 **kw 42) -> bool: 43 """ 44 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 45 46 Parameters 47 ---------- 48 venv: Optional[str], default 'mrsm' 49 The virtual environment to activate. 50 51 color: bool, default True 52 If `True`, include color in debug text. 53 54 force: bool, default False 55 If `True`, do not exit early even if the venv is currently active. 56 57 debug: bool, default False 58 Verbosity toggle. 59 60 Returns 61 ------- 62 A bool indicating whether the virtual environment was successfully activated. 63 64 """ 65 thread_id = get_ident() 66 if active_venvs_order and active_venvs_order[0] == venv: 67 if not force: 68 return True 69 import sys 70 import os 71 if venv is not None: 72 init_venv(venv=venv, debug=debug) 73 with LOCKS['active_venvs']: 74 if thread_id not in threads_active_venvs: 75 threads_active_venvs[thread_id] = {} 76 active_venvs.add(venv) 77 if venv not in threads_active_venvs[thread_id]: 78 threads_active_venvs[thread_id][venv] = 1 79 else: 80 threads_active_venvs[thread_id][venv] += 1 81 82 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 83 if not target_path.exists(): 84 init_venv(venv=venv, force=True, debug=debug) 85 if not target_path.exists(): 86 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 87 target = target_path.as_posix() 88 89 if venv in active_venvs_order: 90 sys.path.remove(target) 91 try: 92 active_venvs_order.remove(venv) 93 except Exception: 94 pass 95 if venv is not None: 96 sys.path.insert(0, target) 97 else: 98 if sys.path and sys.path[0] in (os.getcwd(), ''): 99 sys.path.insert(1, target) 100 else: 101 sys.path.insert(0, target) 102 try: 103 active_venvs_order.insert(0, venv) 104 except Exception: 105 pass 106 107 return True
Create a virtual environment (if it doesn't exist) and add it to sys.path
if necessary.
Parameters
- venv (Optional[str], default 'mrsm'): The virtual environment to activate.
- color (bool, default True):
If
True
, include color in debug text. - force (bool, default False):
If
True
, do not exit early even if the venv is currently active. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment was successfully activated.
110def deactivate_venv( 111 venv: str = 'mrsm', 112 color: bool = True, 113 debug: bool = False, 114 previously_active_venvs: Union['set[str]', List[str], None] = None, 115 force: bool = False, 116 **kw 117) -> bool: 118 """ 119 Remove a virtual environment from `sys.path` (if it's been activated). 120 121 Parameters 122 ---------- 123 venv: str, default 'mrsm' 124 The virtual environment to deactivate. 125 126 color: bool, default True 127 If `True`, include color in debug text. 128 129 debug: bool, default False 130 Verbosity toggle. 131 132 previously_active_venvs: Union[Set[str], List[str], None] 133 If provided, skip deactivating if a virtual environment is in this iterable. 134 135 force: bool, default False 136 If `True`, forcibly deactivate the virtual environment. 137 This may cause issues with other threads, so be careful! 138 139 Returns 140 ------- 141 Return a bool indicating whether the virtual environment was successfully deactivated. 142 143 """ 144 import sys 145 thread_id = get_ident() 146 if venv is None: 147 if venv in active_venvs: 148 active_venvs.remove(venv) 149 return True 150 151 if previously_active_venvs and venv in previously_active_venvs and not force: 152 return True 153 154 with LOCKS['active_venvs']: 155 if venv in threads_active_venvs.get(thread_id, {}): 156 new_count = threads_active_venvs[thread_id][venv] - 1 157 if new_count > 0 and not force: 158 threads_active_venvs[thread_id][venv] = new_count 159 return True 160 else: 161 del threads_active_venvs[thread_id][venv] 162 163 if not force: 164 for other_thread_id, other_venvs in threads_active_venvs.items(): 165 if other_thread_id == thread_id: 166 continue 167 if venv in other_venvs: 168 return True 169 else: 170 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 171 for other_thread_id in to_delete: 172 del threads_active_venvs[other_thread_id] 173 174 if venv in active_venvs: 175 active_venvs.remove(venv) 176 177 if sys.path is None: 178 return False 179 180 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 181 with LOCKS['sys.path']: 182 if target in sys.path: 183 try: 184 sys.path.remove(target) 185 except Exception: 186 pass 187 try: 188 active_venvs_order.remove(venv) 189 except Exception: 190 pass 191 192 return True
Remove a virtual environment from sys.path
(if it's been activated).
Parameters
- venv (str, default 'mrsm'): The virtual environment to deactivate.
- color (bool, default True):
If
True
, include color in debug text. - debug (bool, default False): Verbosity toggle.
- previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
- force (bool, default False):
If
True
, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
- Return a bool indicating whether the virtual environment was successfully deactivated.
822def get_module_venv(module) -> Union[str, None]: 823 """ 824 Return the virtual environment where an imported module is installed. 825 826 Parameters 827 ---------- 828 module: ModuleType 829 The imported module to inspect. 830 831 Returns 832 ------- 833 The name of a venv or `None`. 834 """ 835 import pathlib 836 from meerschaum.config.paths import VIRTENV_RESOURCES_PATH 837 module_path = pathlib.Path(module.__file__).resolve() 838 try: 839 rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH) 840 except ValueError: 841 return None 842 843 return rel_path.as_posix().split('/', maxsplit=1)[0]
Return the virtual environment where an imported module is installed.
Parameters
- module (ModuleType): The imported module to inspect.
Returns
- The name of a venv or
None
.
805def get_venvs() -> List[str]: 806 """ 807 Return a list of all the virtual environments. 808 """ 809 import os 810 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 811 venvs = [] 812 for filename in os.listdir(VIRTENV_RESOURCES_PATH): 813 path = VIRTENV_RESOURCES_PATH / filename 814 if not path.is_dir(): 815 continue 816 if not venv_exists(filename): 817 continue 818 venvs.append(filename) 819 return venvs
Return a list of all the virtual environments.
346def init_venv( 347 venv: str = 'mrsm', 348 verify: bool = True, 349 force: bool = False, 350 debug: bool = False, 351) -> bool: 352 """ 353 Initialize the virtual environment. 354 355 Parameters 356 ---------- 357 venv: str, default 'mrsm' 358 The name of the virtual environment to create. 359 360 verify: bool, default True 361 If `True`, verify that the virtual environment is in the expected state. 362 363 force: bool, default False 364 If `True`, recreate the virtual environment, even if already initalized. 365 366 Returns 367 ------- 368 A `bool` indicating success. 369 """ 370 if not force and venv in verified_venvs: 371 return True 372 if not force and venv_exists(venv, debug=debug): 373 if verify: 374 verify_venv(venv, debug=debug) 375 verified_venvs.add(venv) 376 return True 377 378 import io 379 from contextlib import redirect_stdout 380 import sys 381 import platform 382 import os 383 import pathlib 384 import shutil 385 import time 386 387 from meerschaum.config.static import STATIC_CONFIG 388 from meerschaum.config._paths import ( 389 VIRTENV_RESOURCES_PATH, 390 VENVS_CACHE_RESOURCES_PATH, 391 ) 392 from meerschaum.utils.packages import is_uv_enabled 393 394 venv_path = VIRTENV_RESOURCES_PATH / venv 395 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 396 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 397 lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 398 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 399 400 def update_lock(active: bool): 401 try: 402 if not active: 403 if debug: 404 print(f"Releasing lock: '{lock_path}'") 405 lock_path.unlink() 406 else: 407 if debug: 408 print(f"Acquiring lock: '{lock_path}'") 409 lock_path.touch() 410 except Exception: 411 pass 412 413 def wait_for_lock(): 414 max_lock_seconds = 30.0 415 sleep_message_seconds = 5.0 416 step_sleep_seconds = 0.1 417 init_venv_check_start = time.perf_counter() 418 last_print = init_venv_check_start 419 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 420 if not lock_path.exists(): 421 break 422 423 now = time.perf_counter() 424 if debug or (now - last_print) > sleep_message_seconds: 425 print(f"Lock exists for venv '{venv}', sleeping...") 426 last_print = now 427 time.sleep(step_sleep_seconds) 428 update_lock(False) 429 430 if ( 431 not force 432 and venv == 'mrsm' 433 and os.environ.get(work_dir_env_var, None) is not None 434 and docker_home_venv_path.exists() 435 ): 436 wait_for_lock() 437 shutil.move(docker_home_venv_path, venv_path) 438 if verify: 439 verify_venv(venv, debug=debug) 440 verified_venvs.add(venv) 441 return True 442 443 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 444 global tried_virtualenv 445 try: 446 import venv as _venv 447 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 448 virtualenv = None 449 except ImportError: 450 _venv = None 451 uv = None 452 virtualenv = None 453 454 _venv_success = False 455 temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv) 456 ### NOTE: Disable site-packages movement for now. 457 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 458 459 wait_for_lock() 460 update_lock(True) 461 462 if rename_vtp: 463 if debug: 464 print(f"Moving '{vtp}' to '{temp_vtp}'...") 465 shutil.move(vtp, temp_vtp) 466 467 if uv is not None: 468 _venv_success = run_python_package( 469 'uv', 470 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 471 venv=None, 472 env=_get_pip_os_env(), 473 debug=debug, 474 ) == 0 475 476 if _venv is not None and not _venv_success: 477 f = io.StringIO() 478 with redirect_stdout(f): 479 _venv_success = run_python_package( 480 'venv', 481 [venv_path.as_posix()] + ( 482 ['--symlinks'] 483 if platform.system() != 'Windows' 484 else [] 485 ), 486 venv=None, debug=debug 487 ) == 0 488 if not _venv_success: 489 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 490 if not venv_exists(venv, debug=debug): 491 _venv = None 492 if not _venv_success: 493 virtualenv = attempt_import( 494 'virtualenv', 495 venv=None, 496 lazy=False, 497 install=(not tried_virtualenv), 498 warn=False, 499 check_update=False, 500 color=False, 501 debug=debug, 502 ) 503 if virtualenv is None: 504 print( 505 "Failed to import `venv` or `virtualenv`! " 506 + "Please install `virtualenv` via pip then restart Meerschaum." 507 ) 508 if rename_vtp and temp_vtp.exists(): 509 if debug: 510 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 511 shutil.move(temp_vtp, vtp) 512 update_lock(False) 513 return False 514 515 tried_virtualenv = True 516 try: 517 python_folder = ( 518 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 519 ) 520 dist_packages_path = ( 521 VIRTENV_RESOURCES_PATH / 522 venv / 'local' / 'lib' / python_folder / 'dist-packages' 523 ) 524 local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 525 bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin' 526 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 527 if bin_path.exists(): 528 try: 529 shutil.rmtree(bin_path) 530 except Exception: 531 import traceback 532 traceback.print_exc() 533 virtualenv.cli_run([venv_path.as_posix()]) 534 if dist_packages_path.exists(): 535 vtp.mkdir(exist_ok=True, parents=True) 536 for file_path in dist_packages_path.glob('*'): 537 shutil.move(file_path, vtp) 538 shutil.rmtree(dist_packages_path) 539 # shutil.move(dist_packages_path, vtp) 540 bin_path.mkdir(exist_ok=True, parents=True) 541 for file_path in local_bin_path.glob('*'): 542 shutil.move(file_path, bin_path) 543 # shutil.move(local_bin_path, bin_path) 544 shutil.rmtree(local_bin_path) 545 546 except Exception: 547 import traceback 548 traceback.print_exc() 549 if rename_vtp and temp_vtp.exists(): 550 shutil.move(temp_vtp, vtp) 551 update_lock(False) 552 return False 553 if verify: 554 verify_venv(venv, debug=debug) 555 verified_venvs.add(venv) 556 557 if rename_vtp and temp_vtp.exists(): 558 if debug: 559 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 560 shutil.move(temp_vtp, vtp) 561 562 update_lock(False) 563 return True
Initialize the virtual environment.
Parameters
- venv (str, default 'mrsm'): The name of the virtual environment to create.
- verify (bool, default True):
If
True
, verify that the virtual environment is in the expected state. - force (bool, default False):
If
True
, recreate the virtual environment, even if already initalized.
Returns
- A
bool
indicating success.
792def inside_venv() -> bool: 793 """ 794 Determine whether current Python interpreter is running inside a virtual environment. 795 """ 796 import sys 797 return ( 798 hasattr(sys, 'real_prefix') or ( 799 hasattr(sys, 'base_prefix') 800 and sys.base_prefix != sys.prefix 801 ) 802 )
Determine whether current Python interpreter is running inside a virtual environment.
195def is_venv_active( 196 venv: str = 'mrsm', 197 color : bool = True, 198 debug: bool = False 199) -> bool: 200 """ 201 Check if a virtual environment is active. 202 203 Parameters 204 ---------- 205 venv: str, default 'mrsm' 206 The virtual environment to check. 207 208 color: bool, default True 209 If `True`, include color in debug text. 210 211 debug: bool, default False 212 Verbosity toggle. 213 214 Returns 215 ------- 216 A bool indicating whether the virtual environment `venv` is active. 217 218 """ 219 return venv in active_venvs
Check if a virtual environment is active.
Parameters
- venv (str, default 'mrsm'): The virtual environment to check.
- color (bool, default True):
If
True
, include color in debug text. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment
venv
is active.
591def venv_exec( 592 code: str, 593 venv: Optional[str] = 'mrsm', 594 env: Optional[Dict[str, str]] = None, 595 with_extras: bool = False, 596 as_proc: bool = False, 597 capture_output: bool = True, 598 debug: bool = False, 599) -> Union[bool, Tuple[int, bytes, bytes]]: 600 """ 601 Execute Python code in a subprocess via a virtual environment's interpeter. 602 Return `True` if the code successfully executes, `False` on failure. 603 604 Parameters 605 ---------- 606 code: str 607 The Python code to excecute. 608 609 venv: str, default 'mrsm' 610 The virtual environment to use to get the path for the Python executable. 611 If `venv` is `None`, use the default `sys.executable` path. 612 613 env: Optional[Dict[str, str]], default None 614 Optionally specify the environment variables for the subprocess. 615 Defaults to `os.environ`. 616 617 with_extras: bool, default False 618 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 619 620 as_proc: bool, default False 621 If `True`, return the `subprocess.Popen` object instead of executing. 622 623 Returns 624 ------- 625 By default, return a bool indicating success. 626 If `as_proc` is `True`, return a `subprocess.Popen` object. 627 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 628 629 """ 630 import os 631 import subprocess 632 import platform 633 from meerschaum.utils.debug import dprint 634 executable = venv_executable(venv=venv) 635 cmd_list = [executable, '-c', code] 636 if env is None: 637 env = os.environ 638 if debug: 639 dprint(str(cmd_list)) 640 if not with_extras and not as_proc: 641 return subprocess.call(cmd_list, env=env) == 0 642 643 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 644 group_kwargs = ( 645 { 646 'preexec_fn': os.setsid, 647 } if platform.system() != 'Windows' 648 else { 649 'creationflags': CREATE_NEW_PROCESS_GROUP, 650 } 651 ) 652 process = subprocess.Popen( 653 cmd_list, 654 stdout=stdout, 655 stderr=stderr, 656 env=env, 657 **group_kwargs 658 ) 659 if as_proc: 660 return process 661 stdout, stderr = process.communicate() 662 exit_code = process.returncode 663 return exit_code, stdout, stderr
Execute Python code in a subprocess via a virtual environment's interpeter.
Return True
if the code successfully executes, False
on failure.
Parameters
- code (str): The Python code to excecute.
- venv (str, default 'mrsm'):
The virtual environment to use to get the path for the Python executable.
If
venv
isNone
, use the defaultsys.executable
path. - env (Optional[Dict[str, str]], default None):
Optionally specify the environment variables for the subprocess.
Defaults to
os.environ
. - with_extras (bool, default False):
If
True
, return a tuple of the exit code, stdout bytes, and stderr bytes. - as_proc (bool, default False):
If
True
, return thesubprocess.Popen
object instead of executing.
Returns
- By default, return a bool indicating success.
- If
as_proc
isTrue
, return asubprocess.Popen
object. - If
with_extras
isTrue
, return a tuple of the exit code, stdout bytes, and stderr bytes.
566def venv_executable(venv: Optional[str] = 'mrsm') -> str: 567 """ 568 The Python interpreter executable for a given virtual environment. 569 """ 570 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 571 import sys 572 import platform 573 return ( 574 sys.executable if venv is None 575 else str( 576 VIRTENV_RESOURCES_PATH 577 / venv 578 / ( 579 'bin' if platform.system() != 'Windows' 580 else 'Scripts' 581 ) / ( 582 'python' 583 + str(sys.version_info.major) 584 + '.' 585 + str(sys.version_info.minor) 586 ) 587 ) 588 )
The Python interpreter executable for a given virtual environment.
666def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 667 """ 668 Determine whether a virtual environment has been created. 669 """ 670 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 671 return target_path.exists()
Determine whether a virtual environment has been created.
674def venv_target_path( 675 venv: Union[str, None], 676 allow_nonexistent: bool = False, 677 debug: bool = False, 678) -> 'pathlib.Path': 679 """ 680 Return a virtual environment's site-package path. 681 682 Parameters 683 ---------- 684 venv: Union[str, None] 685 The virtual environment for which a path should be returned. 686 687 allow_nonexistent: bool, default False 688 If `True`, return a path even if it does not exist. 689 690 Returns 691 ------- 692 The `pathlib.Path` object for the virtual environment's path. 693 694 """ 695 import os 696 import sys 697 import platform 698 import pathlib 699 import site 700 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 701 from meerschaum.config.static import STATIC_CONFIG 702 703 ### Check sys.path for a user-writable site-packages directory. 704 if venv is None: 705 706 ### Return the known value for the portable environment. 707 environment_runtime = STATIC_CONFIG['environment']['runtime'] 708 if os.environ.get(environment_runtime, None) == 'portable': 709 python_version_folder = ( 710 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 711 ) 712 executable_path = pathlib.Path(sys.executable) 713 site_packages_path = ( 714 ( 715 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 716 ) if platform.system() != 'Windows' else ( 717 executable_path.parent / 'Lib' / 'site-packages' 718 ) 719 ) 720 if not site_packages_path.exists(): 721 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 722 return site_packages_path 723 724 if not inside_venv(): 725 user_site_packages = site.getusersitepackages() 726 if user_site_packages is None: 727 raise EnvironmentError("Could not determine user site packages.") 728 729 site_path = pathlib.Path(user_site_packages) 730 if not site_path.exists(): 731 732 ### Windows does not have `os.geteuid()`. 733 if platform.system() == 'Windows' or os.geteuid() != 0: 734 site_path.mkdir(parents=True, exist_ok=True) 735 return site_path 736 737 ### Allow for dist-level paths (running as root). 738 for possible_dist in site.getsitepackages(): 739 dist_path = pathlib.Path(possible_dist) 740 if not dist_path.exists(): 741 continue 742 return dist_path 743 744 raise EnvironmentError("Could not determine the dist-packages directory.") 745 746 return site_path 747 748 venv_root_path = ( 749 (VIRTENV_RESOURCES_PATH / venv) 750 if venv is not None else pathlib.Path(sys.prefix) 751 ) 752 target_path = venv_root_path 753 754 ### Ensure 'lib' or 'Lib' exists. 755 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 756 if not allow_nonexistent: 757 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 758 print(f"Failed to find lib directory for virtual environment '{venv}'.") 759 import traceback 760 traceback.print_stack() 761 sys.exit(1) 762 target_path = target_path / lib 763 764 ### Check if a 'python3.x' folder exists. 765 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 766 if target_path.exists(): 767 target_path = ( 768 (target_path / python_folder) if python_folder in os.listdir(target_path) 769 else target_path 770 ) 771 else: 772 target_path = ( 773 (target_path / python_folder) if platform.system() != 'Windows' 774 else target_path 775 ) 776 777 ### Ensure 'site-packages' exists. 778 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 779 target_path = target_path / 'site-packages' 780 else: 781 import traceback 782 traceback.print_stack() 783 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 784 print("This may be because you are using a different Python version.") 785 print("Try deleting the following directory and restarting Meerschaum:") 786 print(VIRTENV_RESOURCES_PATH) 787 sys.exit(1) 788 789 return target_path
Return a virtual environment's site-package path.
Parameters
- venv (Union[str, None]): The virtual environment for which a path should be returned.
- allow_nonexistent (bool, default False):
If
True
, return a path even if it does not exist.
Returns
- The
pathlib.Path
object for the virtual environment's path.
223def verify_venv( 224 venv: str, 225 debug: bool = False, 226) -> None: 227 """ 228 Verify that the virtual environment matches the expected state. 229 """ 230 import pathlib 231 import platform 232 import os 233 import shutil 234 import sys 235 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 236 from meerschaum.utils.process import run_process 237 from meerschaum.utils.misc import make_symlink, is_symlink 238 from meerschaum.utils.warnings import warn 239 240 venv_path = VIRTENV_RESOURCES_PATH / venv 241 bin_path = venv_path / ( 242 'bin' if platform.system() != 'Windows' else "Scripts" 243 ) 244 current_python_versioned_name = ( 245 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 246 + ('' if platform.system() != 'Windows' else '.exe') 247 ) 248 249 if not (bin_path / current_python_versioned_name).exists(): 250 init_venv(venv, verify=False, force=True, debug=debug) 251 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 252 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 253 if not current_python_in_venv_path.exists(): 254 if is_symlink(current_python_in_venv_path): 255 try: 256 current_python_in_venv_path.unlink() 257 except Exception as e: 258 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 259 try: 260 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 261 except Exception: 262 print( 263 f"Unable to create symlink {current_python_in_venv_path} " 264 + f"to {current_python_in_sys_path}." 265 ) 266 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 267 else: 268 files_to_inspect = [current_python_versioned_name] 269 270 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 271 """ 272 Return the version for the python binary at the given path. 273 """ 274 try: 275 ### It might be a broken symlink, so skip on errors. 276 if debug: 277 print(f"Getting python version for {python_path}") 278 proc = run_process( 279 [str(python_path), '-V'], 280 as_proc=True, 281 capture_output=True, 282 ) 283 stdout, stderr = proc.communicate(timeout=1.0) 284 except Exception as e: 285 ### E.g. the symlink may be broken. 286 if is_symlink(python_path): 287 try: 288 python_path.unlink() 289 except Exception as _e: 290 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 291 return None 292 return stdout.decode('utf-8').strip().replace('Python ', '') 293 294 ### Ensure the versions are symlinked correctly. 295 for filename in files_to_inspect: 296 if not filename.startswith('python'): 297 continue 298 python_path = bin_path / filename 299 version = get_python_version(python_path) 300 if version is None: 301 continue 302 try: 303 major_version = version.split('.', maxsplit=1)[0] 304 minor_version = version.split('.', maxsplit=2)[1] 305 except IndexError: 306 return 307 python_versioned_name = ( 308 'python' + major_version + '.' + minor_version 309 + ('' if platform.system() != 'Windows' else '.exe') 310 ) 311 312 ### E.g. python3.10 actually links to Python 3.10. 313 if filename == python_versioned_name: 314 real_path = pathlib.Path(os.path.realpath(python_path)) 315 if not real_path.exists(): 316 try: 317 python_path.unlink() 318 except Exception: 319 pass 320 init_venv(venv, verify=False, force=True, debug=debug) 321 if not python_path.exists(): 322 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 323 324 if python_path == real_path: 325 continue 326 327 try: 328 python_path.unlink() 329 except Exception: 330 pass 331 success, msg = make_symlink(real_path, python_path) 332 if not success: 333 warn(msg, color=False) 334 continue 335 336 python_versioned_path = bin_path / python_versioned_name 337 if python_versioned_path.exists(): 338 ### Avoid circular symlinks. 339 if get_python_version(python_versioned_path) == version: 340 continue 341 python_versioned_path.unlink() 342 shutil.move(python_path, python_versioned_path)
Verify that the virtual environment matches the expected state.