meerschaum.utils.venv
Manage virtual environments.
1#! /usr/bin/env python3 2# -*- coding: utf-8 -*- 3# vim:fenc=utf-8 4 5""" 6Manage virtual environments. 7""" 8 9from __future__ import annotations 10 11from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple 12from meerschaum.utils.threading import RLock, get_ident 13 14__all__ = sorted([ 15 'activate_venv', 'deactivate_venv', 'init_venv', 16 'inside_venv', 'is_venv_active', 'venv_exec', 17 'venv_executable', 'venv_exists', 'venv_target_path', 18 'Venv', 'get_venvs', 'verify_venv', 'get_module_venv', 19]) 20__pdoc__ = {'Venv': True} 21 22LOCKS = { 23 'sys.path': RLock(), 24 'active_venvs': RLock(), 25 'venvs_active_counts': RLock(), 26} 27 28active_venvs = set() 29active_venvs_counts: Dict[str, int] = {} 30active_venvs_order: List[Optional[str]] = [] 31threads_active_venvs: Dict[int, 'set[str]'] = {} 32CREATE_NEW_PROCESS_GROUP = 0x00000200 33 34 35def activate_venv( 36 venv: Optional[str] = 'mrsm', 37 color: bool = True, 38 force: bool = False, 39 debug: bool = False, 40 **kw 41) -> bool: 42 """ 43 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 44 45 Parameters 46 ---------- 47 venv: Optional[str], default 'mrsm' 48 The virtual environment to activate. 49 50 color: bool, default True 51 If `True`, include color in debug text. 52 53 force: bool, default False 54 If `True`, do not exit early even if the venv is currently active. 55 56 debug: bool, default False 57 Verbosity toggle. 58 59 Returns 60 ------- 61 A bool indicating whether the virtual environment was successfully activated. 62 63 """ 64 thread_id = get_ident() 65 if active_venvs_order and active_venvs_order[0] == venv: 66 if not force: 67 return True 68 import sys 69 import os 70 if venv is not None: 71 init_venv(venv=venv, debug=debug) 72 with LOCKS['active_venvs']: 73 if thread_id not in threads_active_venvs: 74 threads_active_venvs[thread_id] = {} 75 active_venvs.add(venv) 76 if venv not in threads_active_venvs[thread_id]: 77 threads_active_venvs[thread_id][venv] = 1 78 else: 79 threads_active_venvs[thread_id][venv] += 1 80 81 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 82 if not target_path.exists(): 83 init_venv(venv=venv, force=True, debug=debug) 84 if not target_path.exists(): 85 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 86 target = target_path.as_posix() 87 88 if venv in active_venvs_order: 89 try: 90 sys.path.remove(target) 91 except Exception: 92 pass 93 try: 94 active_venvs_order.remove(venv) 95 except Exception: 96 pass 97 if venv is not None: 98 sys.path.insert(0, target) 99 else: 100 if sys.path and sys.path[0] in (os.getcwd(), ''): 101 sys.path.insert(1, target) 102 else: 103 sys.path.insert(0, target) 104 try: 105 active_venvs_order.insert(0, venv) 106 except Exception: 107 pass 108 109 return True 110 111 112def deactivate_venv( 113 venv: str = 'mrsm', 114 color: bool = True, 115 debug: bool = False, 116 previously_active_venvs: Union['set[str]', List[str], None] = None, 117 force: bool = False, 118 **kw 119) -> bool: 120 """ 121 Remove a virtual environment from `sys.path` (if it's been activated). 122 123 Parameters 124 ---------- 125 venv: str, default 'mrsm' 126 The virtual environment to deactivate. 127 128 color: bool, default True 129 If `True`, include color in debug text. 130 131 debug: bool, default False 132 Verbosity toggle. 133 134 previously_active_venvs: Union[Set[str], List[str], None] 135 If provided, skip deactivating if a virtual environment is in this iterable. 136 137 force: bool, default False 138 If `True`, forcibly deactivate the virtual environment. 139 This may cause issues with other threads, so be careful! 140 141 Returns 142 ------- 143 Return a bool indicating whether the virtual environment was successfully deactivated. 144 145 """ 146 import sys 147 thread_id = get_ident() 148 if venv is None: 149 if venv in active_venvs: 150 active_venvs.remove(venv) 151 return True 152 153 if previously_active_venvs and venv in previously_active_venvs and not force: 154 return True 155 156 with LOCKS['active_venvs']: 157 if venv in threads_active_venvs.get(thread_id, {}): 158 new_count = threads_active_venvs[thread_id][venv] - 1 159 if new_count > 0 and not force: 160 threads_active_venvs[thread_id][venv] = new_count 161 return True 162 else: 163 del threads_active_venvs[thread_id][venv] 164 165 if not force: 166 for other_thread_id, other_venvs in threads_active_venvs.items(): 167 if other_thread_id == thread_id: 168 continue 169 if venv in other_venvs: 170 return True 171 else: 172 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 173 for other_thread_id in to_delete: 174 del threads_active_venvs[other_thread_id] 175 176 if venv in active_venvs: 177 active_venvs.remove(venv) 178 179 if sys.path is None: 180 return False 181 182 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 183 with LOCKS['sys.path']: 184 if target in sys.path: 185 try: 186 sys.path.remove(target) 187 except Exception: 188 pass 189 try: 190 active_venvs_order.remove(venv) 191 except Exception: 192 pass 193 194 return True 195 196 197def is_venv_active( 198 venv: str = 'mrsm', 199 color : bool = True, 200 debug: bool = False 201) -> bool: 202 """ 203 Check if a virtual environment is active. 204 205 Parameters 206 ---------- 207 venv: str, default 'mrsm' 208 The virtual environment to check. 209 210 color: bool, default True 211 If `True`, include color in debug text. 212 213 debug: bool, default False 214 Verbosity toggle. 215 216 Returns 217 ------- 218 A bool indicating whether the virtual environment `venv` is active. 219 220 """ 221 return venv in active_venvs 222 223 224verified_venvs = set() 225def verify_venv( 226 venv: str, 227 debug: bool = False, 228) -> None: 229 """ 230 Verify that the virtual environment matches the expected state. 231 """ 232 import pathlib 233 import platform 234 import os 235 import shutil 236 import sys 237 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 238 from meerschaum.utils.process import run_process 239 from meerschaum.utils.misc import make_symlink, is_symlink 240 from meerschaum.utils.warnings import warn 241 242 venv_path = VIRTENV_RESOURCES_PATH / venv 243 bin_path = venv_path / ( 244 'bin' if platform.system() != 'Windows' else "Scripts" 245 ) 246 current_python_versioned_name = ( 247 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 248 + ('' if platform.system() != 'Windows' else '.exe') 249 ) 250 251 if not (bin_path / current_python_versioned_name).exists(): 252 init_venv(venv, verify=False, force=True, debug=debug) 253 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 254 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 255 if not current_python_in_venv_path.exists(): 256 if is_symlink(current_python_in_venv_path): 257 try: 258 current_python_in_venv_path.unlink() 259 except Exception as e: 260 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 261 try: 262 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 263 except Exception: 264 print( 265 f"Unable to create symlink {current_python_in_venv_path} " 266 + f"to {current_python_in_sys_path}." 267 ) 268 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 269 else: 270 files_to_inspect = [current_python_versioned_name] 271 272 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 273 """ 274 Return the version for the python binary at the given path. 275 """ 276 try: 277 ### It might be a broken symlink, so skip on errors. 278 if debug: 279 print(f"Getting python version for {python_path}") 280 proc = run_process( 281 [str(python_path), '-V'], 282 as_proc=True, 283 capture_output=True, 284 ) 285 stdout, stderr = proc.communicate(timeout=1.0) 286 except Exception as e: 287 ### E.g. the symlink may be broken. 288 if is_symlink(python_path): 289 try: 290 python_path.unlink() 291 except Exception as _e: 292 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 293 return None 294 return stdout.decode('utf-8').strip().replace('Python ', '') 295 296 ### Ensure the versions are symlinked correctly. 297 for filename in files_to_inspect: 298 if not filename.startswith('python'): 299 continue 300 python_path = bin_path / filename 301 version = get_python_version(python_path) 302 if version is None: 303 continue 304 try: 305 major_version = version.split('.', maxsplit=1)[0] 306 minor_version = version.split('.', maxsplit=2)[1] 307 except IndexError: 308 return 309 python_versioned_name = ( 310 'python' + major_version + '.' + minor_version 311 + ('' if platform.system() != 'Windows' else '.exe') 312 ) 313 314 ### E.g. python3.10 actually links to Python 3.10. 315 if filename == python_versioned_name: 316 real_path = pathlib.Path(os.path.realpath(python_path)) 317 if not real_path.exists(): 318 try: 319 python_path.unlink() 320 except Exception: 321 pass 322 init_venv(venv, verify=False, force=True, debug=debug) 323 if not python_path.exists(): 324 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 325 326 if python_path == real_path: 327 continue 328 329 try: 330 python_path.unlink() 331 except Exception: 332 pass 333 success, msg = make_symlink(real_path, python_path) 334 if not success: 335 warn(msg, color=False) 336 continue 337 338 python_versioned_path = bin_path / python_versioned_name 339 if python_versioned_path.exists(): 340 ### Avoid circular symlinks. 341 if get_python_version(python_versioned_path) == version: 342 continue 343 python_versioned_path.unlink() 344 shutil.move(python_path, python_versioned_path) 345 346 347tried_virtualenv = False 348def init_venv( 349 venv: str = 'mrsm', 350 verify: bool = True, 351 force: bool = False, 352 debug: bool = False, 353) -> bool: 354 """ 355 Initialize the virtual environment. 356 357 Parameters 358 ---------- 359 venv: str, default 'mrsm' 360 The name of the virtual environment to create. 361 362 verify: bool, default True 363 If `True`, verify that the virtual environment is in the expected state. 364 365 force: bool, default False 366 If `True`, recreate the virtual environment, even if already initalized. 367 368 Returns 369 ------- 370 A `bool` indicating success. 371 """ 372 if not force and venv in verified_venvs: 373 return True 374 if not force and venv_exists(venv, debug=debug): 375 if verify: 376 verify_venv(venv, debug=debug) 377 verified_venvs.add(venv) 378 return True 379 380 import io 381 from contextlib import redirect_stdout 382 import sys 383 import platform 384 import os 385 import pathlib 386 import shutil 387 import time 388 389 from meerschaum.config.static import STATIC_CONFIG 390 from meerschaum.config._paths import ( 391 VIRTENV_RESOURCES_PATH, 392 VENVS_CACHE_RESOURCES_PATH, 393 ) 394 from meerschaum.utils.packages import is_uv_enabled 395 396 venv_path = VIRTENV_RESOURCES_PATH / venv 397 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 398 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 399 lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 400 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 401 402 def update_lock(active: bool): 403 try: 404 if not active: 405 if debug: 406 print(f"Releasing lock: '{lock_path}'") 407 lock_path.unlink() 408 else: 409 if debug: 410 print(f"Acquiring lock: '{lock_path}'") 411 lock_path.touch() 412 except Exception: 413 pass 414 415 def wait_for_lock(): 416 if platform.system() == 'Windows': 417 return 418 max_lock_seconds = 30.0 419 sleep_message_seconds = 5.0 420 step_sleep_seconds = 0.1 421 init_venv_check_start = time.perf_counter() 422 last_print = init_venv_check_start 423 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 424 if not lock_path.exists(): 425 break 426 427 now = time.perf_counter() 428 if debug or (now - last_print) > sleep_message_seconds: 429 print(f"Lock exists for venv '{venv}', sleeping...") 430 last_print = now 431 time.sleep(step_sleep_seconds) 432 update_lock(False) 433 434 if ( 435 not force 436 and venv == 'mrsm' 437 and os.environ.get(work_dir_env_var, None) is not None 438 and docker_home_venv_path.exists() 439 ): 440 wait_for_lock() 441 shutil.move(docker_home_venv_path, venv_path) 442 if verify: 443 verify_venv(venv, debug=debug) 444 verified_venvs.add(venv) 445 return True 446 447 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 448 global tried_virtualenv 449 try: 450 import venv as _venv 451 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 452 virtualenv = None 453 except ImportError: 454 _venv = None 455 uv = None 456 virtualenv = None 457 458 _venv_success = False 459 temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv) 460 ### NOTE: Disable site-packages movement for now. 461 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 462 463 wait_for_lock() 464 update_lock(True) 465 466 if rename_vtp: 467 if debug: 468 print(f"Moving '{vtp}' to '{temp_vtp}'...") 469 shutil.move(vtp, temp_vtp) 470 471 if uv is not None: 472 _venv_success = run_python_package( 473 'uv', 474 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 475 venv=None, 476 env=_get_pip_os_env(), 477 debug=debug, 478 ) == 0 479 480 if _venv is not None and not _venv_success: 481 f = io.StringIO() 482 with redirect_stdout(f): 483 _venv_success = run_python_package( 484 'venv', 485 [venv_path.as_posix()] + ( 486 ['--symlinks'] 487 if platform.system() != 'Windows' 488 else [] 489 ), 490 venv=None, debug=debug 491 ) == 0 492 if not _venv_success: 493 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 494 if not venv_exists(venv, debug=debug): 495 _venv = None 496 if not _venv_success: 497 virtualenv = attempt_import( 498 'virtualenv', 499 venv=None, 500 lazy=False, 501 install=(not tried_virtualenv), 502 warn=False, 503 check_update=False, 504 color=False, 505 debug=debug, 506 ) 507 if virtualenv is None: 508 print( 509 "Failed to import `venv` or `virtualenv`! " 510 + "Please install `virtualenv` via pip then restart Meerschaum." 511 ) 512 if rename_vtp and temp_vtp.exists(): 513 if debug: 514 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 515 shutil.move(temp_vtp, vtp) 516 update_lock(False) 517 return False 518 519 tried_virtualenv = True 520 try: 521 python_folder = ( 522 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 523 ) 524 dist_packages_path = ( 525 VIRTENV_RESOURCES_PATH / 526 venv / 'local' / 'lib' / python_folder / 'dist-packages' 527 ) 528 local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 529 bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin' 530 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 531 if bin_path.exists(): 532 try: 533 shutil.rmtree(bin_path) 534 except Exception: 535 import traceback 536 traceback.print_exc() 537 virtualenv.cli_run([venv_path.as_posix()]) 538 if dist_packages_path.exists(): 539 vtp.mkdir(exist_ok=True, parents=True) 540 for file_path in dist_packages_path.glob('*'): 541 shutil.move(file_path, vtp) 542 shutil.rmtree(dist_packages_path) 543 # shutil.move(dist_packages_path, vtp) 544 bin_path.mkdir(exist_ok=True, parents=True) 545 for file_path in local_bin_path.glob('*'): 546 shutil.move(file_path, bin_path) 547 # shutil.move(local_bin_path, bin_path) 548 shutil.rmtree(local_bin_path) 549 550 except Exception: 551 import traceback 552 traceback.print_exc() 553 if rename_vtp and temp_vtp.exists(): 554 shutil.move(temp_vtp, vtp) 555 update_lock(False) 556 return False 557 if verify: 558 verify_venv(venv, debug=debug) 559 verified_venvs.add(venv) 560 561 if rename_vtp and temp_vtp.exists(): 562 if debug: 563 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 564 shutil.move(temp_vtp, vtp) 565 566 update_lock(False) 567 return True 568 569 570def venv_executable(venv: Optional[str] = 'mrsm') -> str: 571 """ 572 The Python interpreter executable for a given virtual environment. 573 """ 574 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 575 import sys 576 import platform 577 return ( 578 sys.executable if venv is None 579 else str( 580 VIRTENV_RESOURCES_PATH 581 / venv 582 / ( 583 'bin' if platform.system() != 'Windows' 584 else 'Scripts' 585 ) / ( 586 'python' 587 + str(sys.version_info.major) 588 + '.' 589 + str(sys.version_info.minor) 590 ) 591 ) 592 ) 593 594 595def venv_exec( 596 code: str, 597 venv: Optional[str] = 'mrsm', 598 env: Optional[Dict[str, str]] = None, 599 with_extras: bool = False, 600 as_proc: bool = False, 601 capture_output: bool = True, 602 debug: bool = False, 603) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 604 """ 605 Execute Python code in a subprocess via a virtual environment's interpeter. 606 Return `True` if the code successfully executes, `False` on failure. 607 608 Parameters 609 ---------- 610 code: str 611 The Python code to excecute. 612 613 venv: str, default 'mrsm' 614 The virtual environment to use to get the path for the Python executable. 615 If `venv` is `None`, use the default `sys.executable` path. 616 617 env: Optional[Dict[str, str]], default None 618 Optionally specify the environment variables for the subprocess. 619 Defaults to `os.environ`. 620 621 with_extras: bool, default False 622 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 623 624 as_proc: bool, default False 625 If `True`, return the `subprocess.Popen` object instead of executing. 626 627 Returns 628 ------- 629 By default, return a bool indicating success. 630 If `as_proc` is `True`, return a `subprocess.Popen` object. 631 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 632 633 """ 634 import os 635 import subprocess 636 import platform 637 from meerschaum.utils.debug import dprint 638 from meerschaum.utils.process import _child_processes 639 640 executable = venv_executable(venv=venv) 641 cmd_list = [executable, '-c', code] 642 if env is None: 643 env = os.environ 644 if debug: 645 dprint(str(cmd_list)) 646 if not with_extras and not as_proc: 647 return subprocess.call(cmd_list, env=env) == 0 648 649 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 650 group_kwargs = ( 651 { 652 'preexec_fn': os.setsid, 653 } if platform.system() != 'Windows' 654 else { 655 'creationflags': CREATE_NEW_PROCESS_GROUP, 656 } 657 ) 658 process = subprocess.Popen( 659 cmd_list, 660 stdout=stdout, 661 stderr=stderr, 662 env=env, 663 **group_kwargs 664 ) 665 if as_proc: 666 _child_processes.append(process) 667 return process 668 stdout, stderr = process.communicate() 669 exit_code = process.returncode 670 return exit_code, stdout, stderr 671 672 673def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 674 """ 675 Determine whether a virtual environment has been created. 676 """ 677 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 678 return target_path.exists() 679 680 681def venv_target_path( 682 venv: Union[str, None], 683 allow_nonexistent: bool = False, 684 debug: bool = False, 685) -> 'pathlib.Path': 686 """ 687 Return a virtual environment's site-package path. 688 689 Parameters 690 ---------- 691 venv: Union[str, None] 692 The virtual environment for which a path should be returned. 693 694 allow_nonexistent: bool, default False 695 If `True`, return a path even if it does not exist. 696 697 Returns 698 ------- 699 The `pathlib.Path` object for the virtual environment's path. 700 701 """ 702 import os 703 import sys 704 import platform 705 import pathlib 706 import site 707 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 708 from meerschaum.config.static import STATIC_CONFIG 709 710 ### Check sys.path for a user-writable site-packages directory. 711 if venv is None: 712 713 ### Return the known value for the portable environment. 714 environment_runtime = STATIC_CONFIG['environment']['runtime'] 715 if os.environ.get(environment_runtime, None) == 'portable': 716 python_version_folder = ( 717 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 718 ) 719 executable_path = pathlib.Path(sys.executable) 720 site_packages_path = ( 721 ( 722 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 723 ) if platform.system() != 'Windows' else ( 724 executable_path.parent / 'Lib' / 'site-packages' 725 ) 726 ) 727 if not site_packages_path.exists(): 728 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 729 return site_packages_path 730 731 if not inside_venv(): 732 user_site_packages = site.getusersitepackages() 733 if user_site_packages is None: 734 raise EnvironmentError("Could not determine user site packages.") 735 736 site_path = pathlib.Path(user_site_packages) 737 if not site_path.exists(): 738 739 ### Windows does not have `os.geteuid()`. 740 if platform.system() == 'Windows' or os.geteuid() != 0: 741 site_path.mkdir(parents=True, exist_ok=True) 742 return site_path 743 744 ### Allow for dist-level paths (running as root). 745 for possible_dist in site.getsitepackages(): 746 dist_path = pathlib.Path(possible_dist) 747 if not dist_path.exists(): 748 continue 749 return dist_path 750 751 raise EnvironmentError("Could not determine the dist-packages directory.") 752 753 return site_path 754 755 venv_root_path = ( 756 (VIRTENV_RESOURCES_PATH / venv) 757 if venv is not None else pathlib.Path(sys.prefix) 758 ) 759 target_path = venv_root_path 760 761 ### Ensure 'lib' or 'Lib' exists. 762 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 763 if not allow_nonexistent: 764 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 765 print(f"Failed to find lib directory for virtual environment '{venv}'.") 766 import traceback 767 traceback.print_stack() 768 sys.exit(1) 769 target_path = target_path / lib 770 771 ### Check if a 'python3.x' folder exists. 772 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 773 if target_path.exists(): 774 target_path = ( 775 (target_path / python_folder) if python_folder in os.listdir(target_path) 776 else target_path 777 ) 778 else: 779 target_path = ( 780 (target_path / python_folder) if platform.system() != 'Windows' 781 else target_path 782 ) 783 784 ### Ensure 'site-packages' exists. 785 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 786 target_path = target_path / 'site-packages' 787 else: 788 import traceback 789 traceback.print_stack() 790 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 791 print("This may be because you are using a different Python version.") 792 print("Try deleting the following directory and restarting Meerschaum:") 793 print(VIRTENV_RESOURCES_PATH) 794 sys.exit(1) 795 796 return target_path 797 798 799def inside_venv() -> bool: 800 """ 801 Determine whether current Python interpreter is running inside a virtual environment. 802 """ 803 import sys 804 return ( 805 hasattr(sys, 'real_prefix') or ( 806 hasattr(sys, 'base_prefix') 807 and sys.base_prefix != sys.prefix 808 ) 809 ) 810 811 812def get_venvs() -> List[str]: 813 """ 814 Return a list of all the virtual environments. 815 """ 816 import os 817 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 818 venvs = [] 819 for filename in os.listdir(VIRTENV_RESOURCES_PATH): 820 path = VIRTENV_RESOURCES_PATH / filename 821 if not path.is_dir(): 822 continue 823 if not venv_exists(filename): 824 continue 825 venvs.append(filename) 826 return venvs 827 828 829def get_module_venv(module) -> Union[str, None]: 830 """ 831 Return the virtual environment where an imported module is installed. 832 833 Parameters 834 ---------- 835 module: ModuleType 836 The imported module to inspect. 837 838 Returns 839 ------- 840 The name of a venv or `None`. 841 """ 842 import pathlib 843 from meerschaum.config.paths import VIRTENV_RESOURCES_PATH 844 module_path = pathlib.Path(module.__file__).resolve() 845 try: 846 rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH) 847 except ValueError: 848 return None 849 850 return rel_path.as_posix().split('/', maxsplit=1)[0] 851 852 853from meerschaum.utils.venv._Venv import Venv
18class Venv: 19 """ 20 Manage a virtual enviroment's activation status. 21 22 Examples 23 -------- 24 >>> from meerschaum.plugins import Plugin 25 >>> with Venv('mrsm') as venv: 26 ... import pandas 27 >>> with Venv(Plugin('noaa')) as venv: 28 ... import requests 29 >>> venv = Venv('mrsm') 30 >>> venv.activate() 31 True 32 >>> venv.deactivate() 33 True 34 >>> 35 """ 36 37 def __init__( 38 self, 39 venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm', 40 debug: bool = False, 41 ) -> None: 42 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 43 ### For some weird threading issue, 44 ### we can't use `isinstance` here. 45 if 'meerschaum.plugins._Plugin' in str(type(venv)): 46 self._venv = venv.name 47 self._activate = venv.activate_venv 48 self._deactivate = venv.deactivate_venv 49 self._kwargs = {} 50 else: 51 self._venv = venv 52 self._activate = activate_venv 53 self._deactivate = deactivate_venv 54 self._kwargs = {'venv': venv} 55 self._debug = debug 56 ### In case someone calls `deactivate()` before `activate()`. 57 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 58 59 60 def activate(self, debug: bool = False) -> bool: 61 """ 62 Activate this virtual environment. 63 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 64 will also be activated. 65 """ 66 from meerschaum.utils.venv import active_venvs, init_venv 67 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 68 try: 69 return self._activate(debug=(debug or self._debug), **self._kwargs) 70 except OSError as e: 71 if not init_venv(self._venv, force=True): 72 raise e 73 return self._activate(debug=(debug or self._debug), **self._kwargs) 74 75 76 def deactivate(self, debug: bool = False) -> bool: 77 """ 78 Deactivate this virtual environment. 79 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 80 will also be deactivated. 81 """ 82 return self._deactivate(debug=(debug or self._debug), **self._kwargs) 83 84 85 @property 86 def target_path(self) -> pathlib.Path: 87 """ 88 Return the target site-packages path for this virtual environment. 89 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 90 (e.g. Python 3.10 and Python 3.7). 91 """ 92 from meerschaum.utils.venv import venv_target_path 93 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug) 94 95 96 @property 97 def root_path(self) -> pathlib.Path: 98 """ 99 Return the top-level path for this virtual environment. 100 """ 101 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 102 if self._venv is None: 103 return self.target_path.parent 104 return VIRTENV_RESOURCES_PATH / self._venv 105 106 107 def __enter__(self) -> None: 108 self.activate(debug=self._debug) 109 110 111 def __exit__(self, exc_type, exc_value, exc_traceback) -> None: 112 self.deactivate(debug=self._debug) 113 114 115 def __str__(self) -> str: 116 quote = "'" if self._venv is not None else "" 117 return "Venv(" + quote + str(self._venv) + quote + ")" 118 119 120 def __repr__(self) -> str: 121 return self.__str__()
Manage a virtual enviroment's activation status.
Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
... import pandas
>>> with Venv(Plugin('noaa')) as venv:
... import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
37 def __init__( 38 self, 39 venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm', 40 debug: bool = False, 41 ) -> None: 42 from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs 43 ### For some weird threading issue, 44 ### we can't use `isinstance` here. 45 if 'meerschaum.plugins._Plugin' in str(type(venv)): 46 self._venv = venv.name 47 self._activate = venv.activate_venv 48 self._deactivate = venv.deactivate_venv 49 self._kwargs = {} 50 else: 51 self._venv = venv 52 self._activate = activate_venv 53 self._deactivate = deactivate_venv 54 self._kwargs = {'venv': venv} 55 self._debug = debug 56 ### In case someone calls `deactivate()` before `activate()`. 57 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
60 def activate(self, debug: bool = False) -> bool: 61 """ 62 Activate this virtual environment. 63 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 64 will also be activated. 65 """ 66 from meerschaum.utils.venv import active_venvs, init_venv 67 self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs) 68 try: 69 return self._activate(debug=(debug or self._debug), **self._kwargs) 70 except OSError as e: 71 if not init_venv(self._venv, force=True): 72 raise e 73 return self._activate(debug=(debug or self._debug), **self._kwargs)
Activate this virtual environment.
If a meerschaum.plugins.Plugin
was provided, its dependent virtual environments
will also be activated.
76 def deactivate(self, debug: bool = False) -> bool: 77 """ 78 Deactivate this virtual environment. 79 If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments 80 will also be deactivated. 81 """ 82 return self._deactivate(debug=(debug or self._debug), **self._kwargs)
Deactivate this virtual environment.
If a meerschaum.plugins.Plugin
was provided, its dependent virtual environments
will also be deactivated.
85 @property 86 def target_path(self) -> pathlib.Path: 87 """ 88 Return the target site-packages path for this virtual environment. 89 A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version 90 (e.g. Python 3.10 and Python 3.7). 91 """ 92 from meerschaum.utils.venv import venv_target_path 93 return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
Return the target site-packages path for this virtual environment.
A meerschaum.utils.venv.Venv
may have one virtual environment per minor Python version
(e.g. Python 3.10 and Python 3.7).
96 @property 97 def root_path(self) -> pathlib.Path: 98 """ 99 Return the top-level path for this virtual environment. 100 """ 101 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 102 if self._venv is None: 103 return self.target_path.parent 104 return VIRTENV_RESOURCES_PATH / self._venv
Return the top-level path for this virtual environment.
36def activate_venv( 37 venv: Optional[str] = 'mrsm', 38 color: bool = True, 39 force: bool = False, 40 debug: bool = False, 41 **kw 42) -> bool: 43 """ 44 Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary. 45 46 Parameters 47 ---------- 48 venv: Optional[str], default 'mrsm' 49 The virtual environment to activate. 50 51 color: bool, default True 52 If `True`, include color in debug text. 53 54 force: bool, default False 55 If `True`, do not exit early even if the venv is currently active. 56 57 debug: bool, default False 58 Verbosity toggle. 59 60 Returns 61 ------- 62 A bool indicating whether the virtual environment was successfully activated. 63 64 """ 65 thread_id = get_ident() 66 if active_venvs_order and active_venvs_order[0] == venv: 67 if not force: 68 return True 69 import sys 70 import os 71 if venv is not None: 72 init_venv(venv=venv, debug=debug) 73 with LOCKS['active_venvs']: 74 if thread_id not in threads_active_venvs: 75 threads_active_venvs[thread_id] = {} 76 active_venvs.add(venv) 77 if venv not in threads_active_venvs[thread_id]: 78 threads_active_venvs[thread_id][venv] = 1 79 else: 80 threads_active_venvs[thread_id][venv] += 1 81 82 target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True) 83 if not target_path.exists(): 84 init_venv(venv=venv, force=True, debug=debug) 85 if not target_path.exists(): 86 raise EnvironmentError(f"Could not activate virtual environment '{venv}'.") 87 target = target_path.as_posix() 88 89 if venv in active_venvs_order: 90 try: 91 sys.path.remove(target) 92 except Exception: 93 pass 94 try: 95 active_venvs_order.remove(venv) 96 except Exception: 97 pass 98 if venv is not None: 99 sys.path.insert(0, target) 100 else: 101 if sys.path and sys.path[0] in (os.getcwd(), ''): 102 sys.path.insert(1, target) 103 else: 104 sys.path.insert(0, target) 105 try: 106 active_venvs_order.insert(0, venv) 107 except Exception: 108 pass 109 110 return True
Create a virtual environment (if it doesn't exist) and add it to sys.path
if necessary.
Parameters
- venv (Optional[str], default 'mrsm'): The virtual environment to activate.
- color (bool, default True):
If
True
, include color in debug text. - force (bool, default False):
If
True
, do not exit early even if the venv is currently active. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment was successfully activated.
113def deactivate_venv( 114 venv: str = 'mrsm', 115 color: bool = True, 116 debug: bool = False, 117 previously_active_venvs: Union['set[str]', List[str], None] = None, 118 force: bool = False, 119 **kw 120) -> bool: 121 """ 122 Remove a virtual environment from `sys.path` (if it's been activated). 123 124 Parameters 125 ---------- 126 venv: str, default 'mrsm' 127 The virtual environment to deactivate. 128 129 color: bool, default True 130 If `True`, include color in debug text. 131 132 debug: bool, default False 133 Verbosity toggle. 134 135 previously_active_venvs: Union[Set[str], List[str], None] 136 If provided, skip deactivating if a virtual environment is in this iterable. 137 138 force: bool, default False 139 If `True`, forcibly deactivate the virtual environment. 140 This may cause issues with other threads, so be careful! 141 142 Returns 143 ------- 144 Return a bool indicating whether the virtual environment was successfully deactivated. 145 146 """ 147 import sys 148 thread_id = get_ident() 149 if venv is None: 150 if venv in active_venvs: 151 active_venvs.remove(venv) 152 return True 153 154 if previously_active_venvs and venv in previously_active_venvs and not force: 155 return True 156 157 with LOCKS['active_venvs']: 158 if venv in threads_active_venvs.get(thread_id, {}): 159 new_count = threads_active_venvs[thread_id][venv] - 1 160 if new_count > 0 and not force: 161 threads_active_venvs[thread_id][venv] = new_count 162 return True 163 else: 164 del threads_active_venvs[thread_id][venv] 165 166 if not force: 167 for other_thread_id, other_venvs in threads_active_venvs.items(): 168 if other_thread_id == thread_id: 169 continue 170 if venv in other_venvs: 171 return True 172 else: 173 to_delete = [other_thread_id for other_thread_id in threads_active_venvs] 174 for other_thread_id in to_delete: 175 del threads_active_venvs[other_thread_id] 176 177 if venv in active_venvs: 178 active_venvs.remove(venv) 179 180 if sys.path is None: 181 return False 182 183 target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix() 184 with LOCKS['sys.path']: 185 if target in sys.path: 186 try: 187 sys.path.remove(target) 188 except Exception: 189 pass 190 try: 191 active_venvs_order.remove(venv) 192 except Exception: 193 pass 194 195 return True
Remove a virtual environment from sys.path
(if it's been activated).
Parameters
- venv (str, default 'mrsm'): The virtual environment to deactivate.
- color (bool, default True):
If
True
, include color in debug text. - debug (bool, default False): Verbosity toggle.
- previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
- force (bool, default False):
If
True
, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
- Return a bool indicating whether the virtual environment was successfully deactivated.
830def get_module_venv(module) -> Union[str, None]: 831 """ 832 Return the virtual environment where an imported module is installed. 833 834 Parameters 835 ---------- 836 module: ModuleType 837 The imported module to inspect. 838 839 Returns 840 ------- 841 The name of a venv or `None`. 842 """ 843 import pathlib 844 from meerschaum.config.paths import VIRTENV_RESOURCES_PATH 845 module_path = pathlib.Path(module.__file__).resolve() 846 try: 847 rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH) 848 except ValueError: 849 return None 850 851 return rel_path.as_posix().split('/', maxsplit=1)[0]
Return the virtual environment where an imported module is installed.
Parameters
- module (ModuleType): The imported module to inspect.
Returns
- The name of a venv or
None
.
813def get_venvs() -> List[str]: 814 """ 815 Return a list of all the virtual environments. 816 """ 817 import os 818 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 819 venvs = [] 820 for filename in os.listdir(VIRTENV_RESOURCES_PATH): 821 path = VIRTENV_RESOURCES_PATH / filename 822 if not path.is_dir(): 823 continue 824 if not venv_exists(filename): 825 continue 826 venvs.append(filename) 827 return venvs
Return a list of all the virtual environments.
349def init_venv( 350 venv: str = 'mrsm', 351 verify: bool = True, 352 force: bool = False, 353 debug: bool = False, 354) -> bool: 355 """ 356 Initialize the virtual environment. 357 358 Parameters 359 ---------- 360 venv: str, default 'mrsm' 361 The name of the virtual environment to create. 362 363 verify: bool, default True 364 If `True`, verify that the virtual environment is in the expected state. 365 366 force: bool, default False 367 If `True`, recreate the virtual environment, even if already initalized. 368 369 Returns 370 ------- 371 A `bool` indicating success. 372 """ 373 if not force and venv in verified_venvs: 374 return True 375 if not force and venv_exists(venv, debug=debug): 376 if verify: 377 verify_venv(venv, debug=debug) 378 verified_venvs.add(venv) 379 return True 380 381 import io 382 from contextlib import redirect_stdout 383 import sys 384 import platform 385 import os 386 import pathlib 387 import shutil 388 import time 389 390 from meerschaum.config.static import STATIC_CONFIG 391 from meerschaum.config._paths import ( 392 VIRTENV_RESOURCES_PATH, 393 VENVS_CACHE_RESOURCES_PATH, 394 ) 395 from meerschaum.utils.packages import is_uv_enabled 396 397 venv_path = VIRTENV_RESOURCES_PATH / venv 398 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 399 docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm') 400 lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock') 401 work_dir_env_var = STATIC_CONFIG['environment']['work_dir'] 402 403 def update_lock(active: bool): 404 try: 405 if not active: 406 if debug: 407 print(f"Releasing lock: '{lock_path}'") 408 lock_path.unlink() 409 else: 410 if debug: 411 print(f"Acquiring lock: '{lock_path}'") 412 lock_path.touch() 413 except Exception: 414 pass 415 416 def wait_for_lock(): 417 if platform.system() == 'Windows': 418 return 419 max_lock_seconds = 30.0 420 sleep_message_seconds = 5.0 421 step_sleep_seconds = 0.1 422 init_venv_check_start = time.perf_counter() 423 last_print = init_venv_check_start 424 while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds): 425 if not lock_path.exists(): 426 break 427 428 now = time.perf_counter() 429 if debug or (now - last_print) > sleep_message_seconds: 430 print(f"Lock exists for venv '{venv}', sleeping...") 431 last_print = now 432 time.sleep(step_sleep_seconds) 433 update_lock(False) 434 435 if ( 436 not force 437 and venv == 'mrsm' 438 and os.environ.get(work_dir_env_var, None) is not None 439 and docker_home_venv_path.exists() 440 ): 441 wait_for_lock() 442 shutil.move(docker_home_venv_path, venv_path) 443 if verify: 444 verify_venv(venv, debug=debug) 445 verified_venvs.add(venv) 446 return True 447 448 from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env 449 global tried_virtualenv 450 try: 451 import venv as _venv 452 uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None 453 virtualenv = None 454 except ImportError: 455 _venv = None 456 uv = None 457 virtualenv = None 458 459 _venv_success = False 460 temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv) 461 ### NOTE: Disable site-packages movement for now. 462 rename_vtp = False and vtp.exists() and not temp_vtp.exists() 463 464 wait_for_lock() 465 update_lock(True) 466 467 if rename_vtp: 468 if debug: 469 print(f"Moving '{vtp}' to '{temp_vtp}'...") 470 shutil.move(vtp, temp_vtp) 471 472 if uv is not None: 473 _venv_success = run_python_package( 474 'uv', 475 ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'], 476 venv=None, 477 env=_get_pip_os_env(), 478 debug=debug, 479 ) == 0 480 481 if _venv is not None and not _venv_success: 482 f = io.StringIO() 483 with redirect_stdout(f): 484 _venv_success = run_python_package( 485 'venv', 486 [venv_path.as_posix()] + ( 487 ['--symlinks'] 488 if platform.system() != 'Windows' 489 else [] 490 ), 491 venv=None, debug=debug 492 ) == 0 493 if not _venv_success: 494 print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...") 495 if not venv_exists(venv, debug=debug): 496 _venv = None 497 if not _venv_success: 498 virtualenv = attempt_import( 499 'virtualenv', 500 venv=None, 501 lazy=False, 502 install=(not tried_virtualenv), 503 warn=False, 504 check_update=False, 505 color=False, 506 debug=debug, 507 ) 508 if virtualenv is None: 509 print( 510 "Failed to import `venv` or `virtualenv`! " 511 + "Please install `virtualenv` via pip then restart Meerschaum." 512 ) 513 if rename_vtp and temp_vtp.exists(): 514 if debug: 515 print(f"Moving '{temp_vtp}' back to '{vtp}'...") 516 shutil.move(temp_vtp, vtp) 517 update_lock(False) 518 return False 519 520 tried_virtualenv = True 521 try: 522 python_folder = ( 523 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 524 ) 525 dist_packages_path = ( 526 VIRTENV_RESOURCES_PATH / 527 venv / 'local' / 'lib' / python_folder / 'dist-packages' 528 ) 529 local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin' 530 bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin' 531 vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug) 532 if bin_path.exists(): 533 try: 534 shutil.rmtree(bin_path) 535 except Exception: 536 import traceback 537 traceback.print_exc() 538 virtualenv.cli_run([venv_path.as_posix()]) 539 if dist_packages_path.exists(): 540 vtp.mkdir(exist_ok=True, parents=True) 541 for file_path in dist_packages_path.glob('*'): 542 shutil.move(file_path, vtp) 543 shutil.rmtree(dist_packages_path) 544 # shutil.move(dist_packages_path, vtp) 545 bin_path.mkdir(exist_ok=True, parents=True) 546 for file_path in local_bin_path.glob('*'): 547 shutil.move(file_path, bin_path) 548 # shutil.move(local_bin_path, bin_path) 549 shutil.rmtree(local_bin_path) 550 551 except Exception: 552 import traceback 553 traceback.print_exc() 554 if rename_vtp and temp_vtp.exists(): 555 shutil.move(temp_vtp, vtp) 556 update_lock(False) 557 return False 558 if verify: 559 verify_venv(venv, debug=debug) 560 verified_venvs.add(venv) 561 562 if rename_vtp and temp_vtp.exists(): 563 if debug: 564 print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.") 565 shutil.move(temp_vtp, vtp) 566 567 update_lock(False) 568 return True
Initialize the virtual environment.
Parameters
- venv (str, default 'mrsm'): The name of the virtual environment to create.
- verify (bool, default True):
If
True
, verify that the virtual environment is in the expected state. - force (bool, default False):
If
True
, recreate the virtual environment, even if already initalized.
Returns
- A
bool
indicating success.
800def inside_venv() -> bool: 801 """ 802 Determine whether current Python interpreter is running inside a virtual environment. 803 """ 804 import sys 805 return ( 806 hasattr(sys, 'real_prefix') or ( 807 hasattr(sys, 'base_prefix') 808 and sys.base_prefix != sys.prefix 809 ) 810 )
Determine whether current Python interpreter is running inside a virtual environment.
198def is_venv_active( 199 venv: str = 'mrsm', 200 color : bool = True, 201 debug: bool = False 202) -> bool: 203 """ 204 Check if a virtual environment is active. 205 206 Parameters 207 ---------- 208 venv: str, default 'mrsm' 209 The virtual environment to check. 210 211 color: bool, default True 212 If `True`, include color in debug text. 213 214 debug: bool, default False 215 Verbosity toggle. 216 217 Returns 218 ------- 219 A bool indicating whether the virtual environment `venv` is active. 220 221 """ 222 return venv in active_venvs
Check if a virtual environment is active.
Parameters
- venv (str, default 'mrsm'): The virtual environment to check.
- color (bool, default True):
If
True
, include color in debug text. - debug (bool, default False): Verbosity toggle.
Returns
- A bool indicating whether the virtual environment
venv
is active.
596def venv_exec( 597 code: str, 598 venv: Optional[str] = 'mrsm', 599 env: Optional[Dict[str, str]] = None, 600 with_extras: bool = False, 601 as_proc: bool = False, 602 capture_output: bool = True, 603 debug: bool = False, 604) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']: 605 """ 606 Execute Python code in a subprocess via a virtual environment's interpeter. 607 Return `True` if the code successfully executes, `False` on failure. 608 609 Parameters 610 ---------- 611 code: str 612 The Python code to excecute. 613 614 venv: str, default 'mrsm' 615 The virtual environment to use to get the path for the Python executable. 616 If `venv` is `None`, use the default `sys.executable` path. 617 618 env: Optional[Dict[str, str]], default None 619 Optionally specify the environment variables for the subprocess. 620 Defaults to `os.environ`. 621 622 with_extras: bool, default False 623 If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 624 625 as_proc: bool, default False 626 If `True`, return the `subprocess.Popen` object instead of executing. 627 628 Returns 629 ------- 630 By default, return a bool indicating success. 631 If `as_proc` is `True`, return a `subprocess.Popen` object. 632 If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes. 633 634 """ 635 import os 636 import subprocess 637 import platform 638 from meerschaum.utils.debug import dprint 639 from meerschaum.utils.process import _child_processes 640 641 executable = venv_executable(venv=venv) 642 cmd_list = [executable, '-c', code] 643 if env is None: 644 env = os.environ 645 if debug: 646 dprint(str(cmd_list)) 647 if not with_extras and not as_proc: 648 return subprocess.call(cmd_list, env=env) == 0 649 650 stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE) 651 group_kwargs = ( 652 { 653 'preexec_fn': os.setsid, 654 } if platform.system() != 'Windows' 655 else { 656 'creationflags': CREATE_NEW_PROCESS_GROUP, 657 } 658 ) 659 process = subprocess.Popen( 660 cmd_list, 661 stdout=stdout, 662 stderr=stderr, 663 env=env, 664 **group_kwargs 665 ) 666 if as_proc: 667 _child_processes.append(process) 668 return process 669 stdout, stderr = process.communicate() 670 exit_code = process.returncode 671 return exit_code, stdout, stderr
Execute Python code in a subprocess via a virtual environment's interpeter.
Return True
if the code successfully executes, False
on failure.
Parameters
- code (str): The Python code to excecute.
- venv (str, default 'mrsm'):
The virtual environment to use to get the path for the Python executable.
If
venv
isNone
, use the defaultsys.executable
path. - env (Optional[Dict[str, str]], default None):
Optionally specify the environment variables for the subprocess.
Defaults to
os.environ
. - with_extras (bool, default False):
If
True
, return a tuple of the exit code, stdout bytes, and stderr bytes. - as_proc (bool, default False):
If
True
, return thesubprocess.Popen
object instead of executing.
Returns
- By default, return a bool indicating success.
- If
as_proc
isTrue
, return asubprocess.Popen
object. - If
with_extras
isTrue
, return a tuple of the exit code, stdout bytes, and stderr bytes.
571def venv_executable(venv: Optional[str] = 'mrsm') -> str: 572 """ 573 The Python interpreter executable for a given virtual environment. 574 """ 575 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 576 import sys 577 import platform 578 return ( 579 sys.executable if venv is None 580 else str( 581 VIRTENV_RESOURCES_PATH 582 / venv 583 / ( 584 'bin' if platform.system() != 'Windows' 585 else 'Scripts' 586 ) / ( 587 'python' 588 + str(sys.version_info.major) 589 + '.' 590 + str(sys.version_info.minor) 591 ) 592 ) 593 )
The Python interpreter executable for a given virtual environment.
674def venv_exists(venv: Union[str, None], debug: bool = False) -> bool: 675 """ 676 Determine whether a virtual environment has been created. 677 """ 678 target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug) 679 return target_path.exists()
Determine whether a virtual environment has been created.
682def venv_target_path( 683 venv: Union[str, None], 684 allow_nonexistent: bool = False, 685 debug: bool = False, 686) -> 'pathlib.Path': 687 """ 688 Return a virtual environment's site-package path. 689 690 Parameters 691 ---------- 692 venv: Union[str, None] 693 The virtual environment for which a path should be returned. 694 695 allow_nonexistent: bool, default False 696 If `True`, return a path even if it does not exist. 697 698 Returns 699 ------- 700 The `pathlib.Path` object for the virtual environment's path. 701 702 """ 703 import os 704 import sys 705 import platform 706 import pathlib 707 import site 708 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 709 from meerschaum.config.static import STATIC_CONFIG 710 711 ### Check sys.path for a user-writable site-packages directory. 712 if venv is None: 713 714 ### Return the known value for the portable environment. 715 environment_runtime = STATIC_CONFIG['environment']['runtime'] 716 if os.environ.get(environment_runtime, None) == 'portable': 717 python_version_folder = ( 718 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 719 ) 720 executable_path = pathlib.Path(sys.executable) 721 site_packages_path = ( 722 ( 723 executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages' 724 ) if platform.system() != 'Windows' else ( 725 executable_path.parent / 'Lib' / 'site-packages' 726 ) 727 ) 728 if not site_packages_path.exists(): 729 raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?") 730 return site_packages_path 731 732 if not inside_venv(): 733 user_site_packages = site.getusersitepackages() 734 if user_site_packages is None: 735 raise EnvironmentError("Could not determine user site packages.") 736 737 site_path = pathlib.Path(user_site_packages) 738 if not site_path.exists(): 739 740 ### Windows does not have `os.geteuid()`. 741 if platform.system() == 'Windows' or os.geteuid() != 0: 742 site_path.mkdir(parents=True, exist_ok=True) 743 return site_path 744 745 ### Allow for dist-level paths (running as root). 746 for possible_dist in site.getsitepackages(): 747 dist_path = pathlib.Path(possible_dist) 748 if not dist_path.exists(): 749 continue 750 return dist_path 751 752 raise EnvironmentError("Could not determine the dist-packages directory.") 753 754 return site_path 755 756 venv_root_path = ( 757 (VIRTENV_RESOURCES_PATH / venv) 758 if venv is not None else pathlib.Path(sys.prefix) 759 ) 760 target_path = venv_root_path 761 762 ### Ensure 'lib' or 'Lib' exists. 763 lib = 'lib' if platform.system() != 'Windows' else 'Lib' 764 if not allow_nonexistent: 765 if not venv_root_path.exists() or lib not in os.listdir(venv_root_path): 766 print(f"Failed to find lib directory for virtual environment '{venv}'.") 767 import traceback 768 traceback.print_stack() 769 sys.exit(1) 770 target_path = target_path / lib 771 772 ### Check if a 'python3.x' folder exists. 773 python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 774 if target_path.exists(): 775 target_path = ( 776 (target_path / python_folder) if python_folder in os.listdir(target_path) 777 else target_path 778 ) 779 else: 780 target_path = ( 781 (target_path / python_folder) if platform.system() != 'Windows' 782 else target_path 783 ) 784 785 ### Ensure 'site-packages' exists. 786 if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows 787 target_path = target_path / 'site-packages' 788 else: 789 import traceback 790 traceback.print_stack() 791 print(f"Failed to find site-packages directory for virtual environment '{venv}'.") 792 print("This may be because you are using a different Python version.") 793 print("Try deleting the following directory and restarting Meerschaum:") 794 print(VIRTENV_RESOURCES_PATH) 795 sys.exit(1) 796 797 return target_path
Return a virtual environment's site-package path.
Parameters
- venv (Union[str, None]): The virtual environment for which a path should be returned.
- allow_nonexistent (bool, default False):
If
True
, return a path even if it does not exist.
Returns
- The
pathlib.Path
object for the virtual environment's path.
226def verify_venv( 227 venv: str, 228 debug: bool = False, 229) -> None: 230 """ 231 Verify that the virtual environment matches the expected state. 232 """ 233 import pathlib 234 import platform 235 import os 236 import shutil 237 import sys 238 from meerschaum.config._paths import VIRTENV_RESOURCES_PATH 239 from meerschaum.utils.process import run_process 240 from meerschaum.utils.misc import make_symlink, is_symlink 241 from meerschaum.utils.warnings import warn 242 243 venv_path = VIRTENV_RESOURCES_PATH / venv 244 bin_path = venv_path / ( 245 'bin' if platform.system() != 'Windows' else "Scripts" 246 ) 247 current_python_versioned_name = ( 248 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor) 249 + ('' if platform.system() != 'Windows' else '.exe') 250 ) 251 252 if not (bin_path / current_python_versioned_name).exists(): 253 init_venv(venv, verify=False, force=True, debug=debug) 254 current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv)) 255 current_python_in_sys_path = pathlib.Path(venv_executable(venv=None)) 256 if not current_python_in_venv_path.exists(): 257 if is_symlink(current_python_in_venv_path): 258 try: 259 current_python_in_venv_path.unlink() 260 except Exception as e: 261 print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}") 262 try: 263 make_symlink(current_python_in_sys_path, current_python_in_venv_path) 264 except Exception: 265 print( 266 f"Unable to create symlink {current_python_in_venv_path} " 267 + f"to {current_python_in_sys_path}." 268 ) 269 files_to_inspect = sorted(os.listdir(bin_path), reverse=True) 270 else: 271 files_to_inspect = [current_python_versioned_name] 272 273 def get_python_version(python_path: pathlib.Path) -> Union[str, None]: 274 """ 275 Return the version for the python binary at the given path. 276 """ 277 try: 278 ### It might be a broken symlink, so skip on errors. 279 if debug: 280 print(f"Getting python version for {python_path}") 281 proc = run_process( 282 [str(python_path), '-V'], 283 as_proc=True, 284 capture_output=True, 285 ) 286 stdout, stderr = proc.communicate(timeout=1.0) 287 except Exception as e: 288 ### E.g. the symlink may be broken. 289 if is_symlink(python_path): 290 try: 291 python_path.unlink() 292 except Exception as _e: 293 print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}") 294 return None 295 return stdout.decode('utf-8').strip().replace('Python ', '') 296 297 ### Ensure the versions are symlinked correctly. 298 for filename in files_to_inspect: 299 if not filename.startswith('python'): 300 continue 301 python_path = bin_path / filename 302 version = get_python_version(python_path) 303 if version is None: 304 continue 305 try: 306 major_version = version.split('.', maxsplit=1)[0] 307 minor_version = version.split('.', maxsplit=2)[1] 308 except IndexError: 309 return 310 python_versioned_name = ( 311 'python' + major_version + '.' + minor_version 312 + ('' if platform.system() != 'Windows' else '.exe') 313 ) 314 315 ### E.g. python3.10 actually links to Python 3.10. 316 if filename == python_versioned_name: 317 real_path = pathlib.Path(os.path.realpath(python_path)) 318 if not real_path.exists(): 319 try: 320 python_path.unlink() 321 except Exception: 322 pass 323 init_venv(venv, verify=False, force=True, debug=debug) 324 if not python_path.exists(): 325 raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}") 326 327 if python_path == real_path: 328 continue 329 330 try: 331 python_path.unlink() 332 except Exception: 333 pass 334 success, msg = make_symlink(real_path, python_path) 335 if not success: 336 warn(msg, color=False) 337 continue 338 339 python_versioned_path = bin_path / python_versioned_name 340 if python_versioned_path.exists(): 341 ### Avoid circular symlinks. 342 if get_python_version(python_versioned_path) == version: 343 continue 344 python_versioned_path.unlink() 345 shutil.move(python_path, python_versioned_path)
Verify that the virtual environment matches the expected state.