meerschaum.utils.venv

Manage virtual environments.

  1#! /usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3# vim:fenc=utf-8
  4
  5"""
  6Manage virtual environments.
  7"""
  8
  9from __future__ import annotations
 10
 11import sys
 12import pathlib
 13
 14from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple
 15from meerschaum.utils.threading import RLock, get_ident
 16
 17__all__ = sorted([
 18    'activate_venv', 'deactivate_venv', 'init_venv',
 19    'inside_venv', 'is_venv_active', 'venv_exec',
 20    'venv_executable', 'venv_exists', 'venv_target_path',
 21    'Venv', 'get_venvs', 'verify_venv', 'get_module_venv',
 22])
 23__pdoc__ = {'Venv': True}
 24
 25LOCKS = {
 26    'sys.path': RLock(),
 27    'active_venvs': RLock(),
 28    'venvs_active_counts': RLock(),
 29}
 30
 31active_venvs = set()
 32active_venvs_counts: Dict[str, int] = {}
 33active_venvs_order: List[Optional[str]] = []
 34threads_active_venvs: Dict[int, 'set[str]'] = {}
 35CREATE_NEW_PROCESS_GROUP = 0x00000200
 36
 37
 38def activate_venv(
 39    venv: Optional[str] = 'mrsm',
 40    color: bool = True,
 41    force: bool = False,
 42    init_if_not_exists: bool = True,
 43    debug: bool = False,
 44    **kw
 45) -> bool:
 46    """
 47    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 48
 49    Parameters
 50    ----------
 51    venv: Optional[str], default 'mrsm'
 52        The virtual environment to activate.
 53
 54    color: bool, default True
 55        If `True`, include color in debug text.
 56
 57    init_if_not_exists: bool, default True
 58        If `True`, create the virtual environment if it does not exist.
 59
 60    force: bool, default False
 61        If `True`, do not exit early even if the venv is currently active.
 62
 63    debug: bool, default False
 64        Verbosity toggle.
 65
 66    Returns
 67    -------
 68    A bool indicating whether the virtual environment was successfully activated.
 69
 70    """
 71    thread_id = get_ident()
 72    if active_venvs_order and active_venvs_order[0] == venv:
 73        if not force:
 74            return True
 75
 76    import sys
 77    import os
 78    if venv is not None and init_if_not_exists:
 79        init_venv(venv=venv, debug=debug)
 80
 81    with LOCKS['active_venvs']:
 82        if thread_id not in threads_active_venvs:
 83            threads_active_venvs[thread_id] = {}
 84        active_venvs.add(venv)
 85        if venv not in threads_active_venvs[thread_id]:
 86            threads_active_venvs[thread_id][venv] = 1
 87        else:
 88            threads_active_venvs[thread_id][venv] += 1
 89
 90        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 91        if not target_path.exists() and venv is not None and init_if_not_exists:
 92            init_venv(venv=venv, force=True, debug=debug)
 93
 94        if not target_path.exists():
 95            if init_if_not_exists:
 96                return False
 97            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 98
 99        target = target_path.as_posix()
100
101        if venv in active_venvs_order:
102            try:
103                sys.path.remove(target)
104            except Exception:
105                pass
106            try:
107                active_venvs_order.remove(venv)
108            except Exception:
109                pass
110
111        if venv is not None:
112            sys.path.insert(0, target)
113        else:
114            if sys.path and sys.path[0] in (os.getcwd(), ''):
115                sys.path.insert(1, target)
116            else:
117                sys.path.insert(0, target)
118        try:
119            active_venvs_order.insert(0, venv)
120        except Exception:
121            pass
122
123    return True
124
125
126def deactivate_venv(
127    venv: str = 'mrsm',
128    color: bool = True,
129    debug: bool = False,
130    previously_active_venvs: Union['set[str]', List[str], None] = None,
131    force: bool = False,
132    **kw
133) -> bool:
134    """
135    Remove a virtual environment from `sys.path` (if it's been activated).
136
137    Parameters
138    ----------
139    venv: str, default 'mrsm'
140        The virtual environment to deactivate.
141
142    color: bool, default True
143        If `True`, include color in debug text.
144
145    debug: bool, default False
146        Verbosity toggle.
147
148    previously_active_venvs: Union[Set[str], List[str], None]
149        If provided, skip deactivating if a virtual environment is in this iterable.
150
151    force: bool, default False
152        If `True`, forcibly deactivate the virtual environment.
153        This may cause issues with other threads, so be careful!
154
155    Returns
156    -------
157    Return a bool indicating whether the virtual environment was successfully deactivated.
158
159    """
160    import sys
161    thread_id = get_ident()
162    if venv is None:
163        if venv in active_venvs:
164            active_venvs.remove(venv)
165        return True
166
167    if previously_active_venvs and venv in previously_active_venvs and not force:
168        return True
169
170    with LOCKS['active_venvs']:
171        if venv in threads_active_venvs.get(thread_id, {}):
172            new_count = threads_active_venvs[thread_id][venv] - 1
173            if new_count > 0 and not force:
174                threads_active_venvs[thread_id][venv] = new_count
175                return True
176            else:
177                del threads_active_venvs[thread_id][venv]
178
179        if not force:
180            for other_thread_id, other_venvs in threads_active_venvs.items():
181                if other_thread_id == thread_id:
182                    continue
183                if venv in other_venvs:
184                    return True
185        else:
186            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
187            for other_thread_id in to_delete:
188                del threads_active_venvs[other_thread_id]
189
190        if venv in active_venvs:
191            active_venvs.remove(venv)
192
193    if sys.path is None:
194        return False
195
196    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
197    with LOCKS['sys.path']:
198        if target in sys.path:
199            try:
200                sys.path.remove(target)
201            except Exception:
202                pass
203            try:
204                active_venvs_order.remove(venv)
205            except Exception:
206                pass
207
208    return True
209
210
211def is_venv_active(
212    venv: str = 'mrsm',
213    color : bool = True,
214    debug: bool = False
215) -> bool:
216    """
217    Check if a virtual environment is active.
218
219    Parameters
220    ----------
221    venv: str, default 'mrsm'
222        The virtual environment to check.
223
224    color: bool, default True
225        If `True`, include color in debug text.
226
227    debug: bool, default False
228        Verbosity toggle.
229
230    Returns
231    -------
232    A bool indicating whether the virtual environment `venv` is active.
233
234    """
235    return venv in active_venvs
236
237
238verified_venvs = set()
239def verify_venv(
240    venv: str,
241    debug: bool = False,
242) -> None:
243    """
244    Verify that the virtual environment matches the expected state.
245    """
246    import platform
247    import os
248    import shutil
249    import sys
250
251    import meerschaum.config.paths as paths
252    from meerschaum.utils.process import run_process
253    from meerschaum.utils.misc import make_symlink, is_symlink
254    from meerschaum.utils.warnings import warn
255
256    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
257    bin_path = venv_path / (
258        'bin' if platform.system() != 'Windows' else "Scripts"
259    )
260    current_python_versioned_name = (
261        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
262        + ('' if platform.system() != 'Windows' else '.exe')
263    )
264
265    if not (bin_path / current_python_versioned_name).exists():
266        init_venv(venv, verify=False, force=True, debug=debug)
267        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
268        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
269        if not current_python_in_venv_path.exists():
270            if is_symlink(current_python_in_venv_path):
271                try:
272                    current_python_in_venv_path.unlink()
273                except Exception as e:
274                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
275            try:
276                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
277            except Exception:
278                print(
279                    f"Unable to create symlink {current_python_in_venv_path} "
280                    + f"to {current_python_in_sys_path}."
281                )
282        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
283    else:
284        files_to_inspect = [current_python_versioned_name]
285
286    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
287        """
288        Return the version for the python binary at the given path.
289        """
290        try:
291            ### It might be a broken symlink, so skip on errors.
292            if debug:
293                print(f"Getting python version for {python_path}")
294            proc = run_process(
295                [str(python_path), '-V'],
296                as_proc=True,
297                capture_output=True,
298            )
299            stdout, stderr = proc.communicate(timeout=1.0)
300        except Exception as e:
301            ### E.g. the symlink may be broken.
302            if is_symlink(python_path):
303                try:
304                    python_path.unlink()
305                except Exception as _e:
306                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
307            return None
308        return stdout.decode('utf-8').strip().replace('Python ', '')
309
310    ### Ensure the versions are symlinked correctly.
311    for filename in files_to_inspect:
312        if not filename.startswith('python'):
313            continue
314        python_path = bin_path / filename
315        version = get_python_version(python_path)
316        if version is None:
317            continue
318        try:
319            major_version = version.split('.', maxsplit=1)[0]
320            minor_version = version.split('.', maxsplit=2)[1]
321        except IndexError:
322            return
323        python_versioned_name = (
324            'python' + major_version + '.' + minor_version
325            + ('' if platform.system() != 'Windows' else '.exe')
326        )
327
328        ### E.g. python3.10 actually links to Python 3.10.
329        if filename == python_versioned_name:
330            try:
331                real_path = pathlib.Path(os.path.realpath(python_path))
332                real_path_exists = real_path.exists()
333            except Exception:
334                real_path_exists = False
335
336            if not real_path_exists:
337                try:
338                    python_path.unlink()
339                except Exception:
340                    pass
341                init_venv(venv, verify=False, force=True, debug=debug)
342                if not python_path.exists():
343                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
344
345            if python_path == real_path:
346                continue
347
348            try:
349                python_path.unlink()
350            except Exception:
351                pass
352            success, msg = make_symlink(real_path, python_path)
353            if not success:
354                warn(msg, color=False)
355            continue
356
357        python_versioned_path = bin_path / python_versioned_name
358        if python_versioned_path.exists():
359            ### Avoid circular symlinks.
360            if get_python_version(python_versioned_path) == version:
361                continue
362            python_versioned_path.unlink()
363        shutil.move(python_path, python_versioned_path)
364
365
366tried_virtualenv = False
367def init_venv(
368    venv: str = 'mrsm',
369    verify: bool = True,
370    force: bool = False,
371    debug: bool = False,
372) -> bool:
373    """
374    Initialize the virtual environment.
375
376    Parameters
377    ----------
378    venv: str, default 'mrsm'
379        The name of the virtual environment to create.
380
381    verify: bool, default True
382        If `True`, verify that the virtual environment is in the expected state.
383
384    force: bool, default False
385        If `True`, recreate the virtual environment, even if already initalized.
386
387    Returns
388    -------
389    A `bool` indicating success.
390    """
391    if not force and venv in verified_venvs:
392        return True
393    if not force and venv_exists(venv, debug=debug):
394        if verify:
395            verify_venv(venv, debug=debug)
396            verified_venvs.add(venv)
397        return True
398
399    import io
400    from contextlib import redirect_stdout
401    import sys
402    import platform
403    import os
404    import shutil
405    import time
406
407    import meerschaum.config.paths as paths
408    from meerschaum._internal.static import STATIC_CONFIG
409    from meerschaum.utils.packages import is_uv_enabled
410
411    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
412    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
413    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
414    lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
415    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
416
417    def update_lock(active: bool):
418        try:
419            if not active:
420                if debug:
421                    print(f"Releasing lock: '{lock_path}'")
422                lock_path.unlink()
423            else:
424                if debug:
425                    print(f"Acquiring lock: '{lock_path}'")
426                lock_path.touch()
427        except Exception:
428            pass
429
430    def wait_for_lock():
431        if platform.system() == 'Windows':
432            return
433        max_lock_seconds = 30.0
434        sleep_message_seconds = 5.0
435        step_sleep_seconds = 0.1
436        init_venv_check_start = time.perf_counter()
437        last_print = init_venv_check_start
438        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
439            if not lock_path.exists():
440                break
441
442            now = time.perf_counter()
443            if debug or (now - last_print) > sleep_message_seconds:
444                print(f"Lock exists for venv '{venv}', sleeping...")
445                last_print = now
446            time.sleep(step_sleep_seconds)
447        update_lock(False)
448
449    if (
450        not force
451        and venv == 'mrsm'
452        and os.environ.get(work_dir_env_var, None) is not None
453        and docker_home_venv_path.exists()
454    ):
455        wait_for_lock()
456        shutil.move(docker_home_venv_path, venv_path)
457        if verify:
458            verify_venv(venv, debug=debug)
459            verified_venvs.add(venv)
460        return True
461
462    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
463    global tried_virtualenv
464    try:
465        import venv as _venv
466        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
467        virtualenv = None
468    except ImportError:
469        _venv = None
470        uv = None
471        virtualenv = None
472
473    _venv_success = False
474    temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv)
475    ### NOTE: Disable site-packages movement for now.
476    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
477
478    wait_for_lock()
479    update_lock(True)
480
481    if rename_vtp:
482        if debug:
483            print(f"Moving '{vtp}' to '{temp_vtp}'...")
484        shutil.move(vtp, temp_vtp)
485
486    if uv is not None:
487        _venv_success = run_python_package(
488            'uv',
489            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
490            venv=None,
491            env=_get_pip_os_env(),
492            debug=debug,
493        ) == 0
494
495    if _venv is not None and not _venv_success:
496        f = io.StringIO()
497        with redirect_stdout(f):
498            _venv_success = run_python_package(
499                'venv',
500                [venv_path.as_posix()] + (
501                    ['--symlinks']
502                    if platform.system() != 'Windows'
503                    else []
504                ),
505                venv=None, debug=debug
506            ) == 0
507        if not _venv_success:
508            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
509        if not venv_exists(venv, debug=debug):
510            _venv = None
511    if not _venv_success:
512        virtualenv = attempt_import(
513            'virtualenv',
514            venv=None,
515            lazy=False,
516            install=(not tried_virtualenv),
517            warn=False,
518            check_update=False,
519            color=False,
520            debug=debug,
521        )
522        if virtualenv is None:
523            print(
524                "Failed to import `venv` or `virtualenv`! "
525                + "Please install `virtualenv` via pip then restart Meerschaum."
526            )
527            if rename_vtp and temp_vtp.exists():
528                if debug:
529                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
530                shutil.move(temp_vtp, vtp)
531            update_lock(False)
532            return False
533
534        tried_virtualenv = True
535        try:
536            python_folder = (
537                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
538            )
539            dist_packages_path = (
540                paths.VIRTENV_RESOURCES_PATH /
541                venv / 'local' / 'lib' / python_folder / 'dist-packages'
542            )
543            local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
544            bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin'
545            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
546            if bin_path.exists():
547                try:
548                    shutil.rmtree(bin_path)
549                except Exception:
550                    import traceback
551                    traceback.print_exc()
552            virtualenv.cli_run([venv_path.as_posix()])
553            if dist_packages_path.exists():
554                vtp.mkdir(exist_ok=True, parents=True)
555                for file_path in dist_packages_path.glob('*'):
556                    shutil.move(file_path, vtp)
557                shutil.rmtree(dist_packages_path)
558                #  shutil.move(dist_packages_path, vtp)
559                bin_path.mkdir(exist_ok=True, parents=True)
560                for file_path in local_bin_path.glob('*'):
561                    shutil.move(file_path, bin_path)
562                #  shutil.move(local_bin_path, bin_path)
563                shutil.rmtree(local_bin_path)
564
565        except Exception:
566            import traceback
567            traceback.print_exc()
568            if rename_vtp and temp_vtp.exists():
569                shutil.move(temp_vtp, vtp)
570            update_lock(False)
571            return False
572    if verify:
573        verify_venv(venv, debug=debug)
574        verified_venvs.add(venv)
575
576    if rename_vtp and temp_vtp.exists():
577        if debug:
578            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
579        shutil.move(temp_vtp, vtp)
580
581    update_lock(False)
582    return True
583
584
585def venv_executable(venv: Optional[str] = 'mrsm') -> str:
586    """
587    The Python interpreter executable for a given virtual environment.
588    """
589    import sys
590    import platform
591    import meerschaum.config.paths as paths
592    return (
593        sys.executable if venv is None
594        else str(
595            paths.VIRTENV_RESOURCES_PATH
596            / venv
597            / (
598                'bin' if platform.system() != 'Windows'
599                else 'Scripts'
600            ) / (
601                'python'
602                + str(sys.version_info.major)
603                + '.'
604                + str(sys.version_info.minor)
605            )
606        )
607    )
608
609
610def venv_exec(
611    code: str,
612    venv: Optional[str] = 'mrsm',
613    env: Optional[Dict[str, str]] = None,
614    with_extras: bool = False,
615    as_proc: bool = False,
616    capture_output: bool = True,
617    debug: bool = False,
618) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
619    """
620    Execute Python code in a subprocess via a virtual environment's interpeter.
621    Return `True` if the code successfully executes, `False` on failure.
622
623    Parameters
624    ----------
625    code: str
626        The Python code to excecute.
627
628    venv: str, default 'mrsm'
629        The virtual environment to use to get the path for the Python executable.
630        If `venv` is `None`, use the default `sys.executable` path.
631
632    env: Optional[Dict[str, str]], default None
633        Optionally specify the environment variables for the subprocess.
634        Defaults to `os.environ`.
635
636    with_extras: bool, default False
637        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
638
639    as_proc: bool, default False
640        If `True`, return the `subprocess.Popen` object instead of executing.
641
642    Returns
643    -------
644    By default, return a bool indicating success.
645    If `as_proc` is `True`, return a `subprocess.Popen` object.
646    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
647
648    """
649    import os
650    import subprocess
651    import platform
652    from meerschaum.utils.debug import dprint
653    from meerschaum.utils.process import _child_processes
654
655    executable = venv_executable(venv=venv)
656    cmd_list = [executable, '-c', code]
657    if env is None:
658        env = os.environ
659    if debug:
660        dprint(str(cmd_list))
661    if not with_extras and not as_proc:
662        return subprocess.call(cmd_list, env=env) == 0
663
664    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
665    group_kwargs = (
666        {
667            'preexec_fn': os.setsid,
668        } if platform.system() != 'Windows'
669        else {
670            'creationflags': CREATE_NEW_PROCESS_GROUP,
671        }
672    )
673    process = subprocess.Popen(
674        cmd_list,
675        stdout=stdout,
676        stderr=stderr,
677        stdin=sys.stdin,
678        env=env,
679        **group_kwargs
680    )
681    if as_proc:
682        _child_processes.append(process)
683        return process
684    stdout, stderr = process.communicate()
685    exit_code = process.returncode
686    return exit_code, stdout, stderr
687
688
689def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
690    """
691    Determine whether a virtual environment has been created.
692    """
693    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
694    return target_path.exists()
695
696
697def venv_target_path(
698    venv: Union[str, None],
699    allow_nonexistent: bool = False,
700    debug: bool = False,
701) -> pathlib.Path:
702    """
703    Return a virtual environment's site-package path.
704
705    Parameters
706    ----------
707    venv: Union[str, None]
708        The virtual environment for which a path should be returned.
709
710    allow_nonexistent: bool, default False
711        If `True`, return a path even if it does not exist.
712
713    Returns
714    -------
715    The `pathlib.Path` object for the virtual environment's path.
716
717    """
718    import os
719    import sys
720    import platform
721    import site
722
723    import meerschaum.config.paths as paths
724    from meerschaum._internal.static import STATIC_CONFIG
725
726    ### Check sys.path for a user-writable site-packages directory.
727    if venv is None:
728
729        ### Return the known value for the portable environment.
730        environment_runtime = STATIC_CONFIG['environment']['runtime']
731        if os.environ.get(environment_runtime, None) == 'portable':
732            python_version_folder = (
733                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
734            )
735            executable_path = pathlib.Path(sys.executable)
736            site_packages_path = (
737                (
738                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
739                ) if platform.system() != 'Windows' else (
740                    executable_path.parent / 'Lib' / 'site-packages'
741                )
742            )
743            if not site_packages_path.exists():
744                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
745            return site_packages_path
746
747        if not inside_venv():
748            user_site_packages = site.getusersitepackages()
749            if user_site_packages is None:
750                raise EnvironmentError("Could not determine user site packages.")
751
752            site_path = pathlib.Path(user_site_packages)
753            if not site_path.exists():
754
755                ### Windows does not have `os.geteuid()`.
756                if platform.system() == 'Windows' or os.geteuid() != 0:
757                    site_path.mkdir(parents=True, exist_ok=True)
758                    return site_path
759
760                ### Allow for dist-level paths (running as root).
761                for possible_dist in site.getsitepackages():
762                    dist_path = pathlib.Path(possible_dist)
763                    if not dist_path.exists():
764                        continue
765                    return dist_path
766                
767                raise EnvironmentError("Could not determine the dist-packages directory.")
768
769            return site_path
770
771    venv_root_path = (
772        (paths.VIRTENV_RESOURCES_PATH / venv)
773        if venv is not None
774        else pathlib.Path(sys.prefix)
775    )
776    target_path = venv_root_path
777
778    ### Ensure 'lib' or 'Lib' exists.
779    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
780    if not allow_nonexistent:
781        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
782            print(f"Failed to find lib directory for virtual environment '{venv}'.")
783            import traceback
784            traceback.print_stack()
785            sys.exit(1)
786    target_path = target_path / lib
787
788    ### Check if a 'python3.x' folder exists.
789    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
790    if target_path.exists():
791        target_path = (
792            (target_path / python_folder)
793            if python_folder in os.listdir(target_path)
794            else target_path
795        )
796    else:
797        target_path = (
798            (target_path / python_folder)
799            if platform.system() != 'Windows'
800            else target_path
801        )
802
803    ### Ensure 'site-packages' exists.
804    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
805        target_path = target_path / 'site-packages'
806    else:
807        import traceback
808        traceback.print_stack()
809        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
810        print("This may be because you are using a different Python version.")
811        print("Try deleting the following directory and restarting Meerschaum:")
812        print(paths.VIRTENV_RESOURCES_PATH)
813        sys.exit(1)
814
815    return target_path
816
817
818def inside_venv() -> bool:
819    """
820    Determine whether current Python interpreter is running inside a virtual environment.
821    """
822    import sys
823    return (
824        hasattr(sys, 'real_prefix') or (
825            hasattr(sys, 'base_prefix')
826                and sys.base_prefix != sys.prefix
827        )
828    )
829
830
831def get_venvs() -> List[str]:
832    """
833    Return a list of all the virtual environments.
834    """
835    import os
836    import meerschaum.config.paths as paths
837    venvs = []
838    for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH):
839        path = paths.VIRTENV_RESOURCES_PATH / filename
840        if not path.is_dir():
841            continue
842        if not venv_exists(filename):
843            continue
844        venvs.append(filename)
845    return venvs
846
847
848def get_module_venv(module) -> Union[str, None]:
849    """
850    Return the virtual environment where an imported module is installed.
851
852    Parameters
853    ----------
854    module: ModuleType
855        The imported module to inspect.
856
857    Returns
858    -------
859    The name of a venv or `None`.
860    """
861    import meerschaum.config.paths as paths
862    module_path = pathlib.Path(module.__file__).resolve()
863    try:
864        rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH)
865    except ValueError:
866        return None
867
868    return rel_path.as_posix().split('/', maxsplit=1)[0]
869
870
871from meerschaum.utils.venv._Venv import Venv
class Venv:
 19class Venv:
 20    """
 21    Manage a virtual enviroment's activation status.
 22
 23    Examples
 24    --------
 25    >>> from meerschaum.plugins import Plugin
 26    >>> with Venv('mrsm') as venv:
 27    ...     import pandas
 28    >>> with Venv(Plugin('noaa')) as venv:
 29    ...     import requests
 30    >>> venv = Venv('mrsm')
 31    >>> venv.activate()
 32    True
 33    >>> venv.deactivate()
 34    True
 35    >>> 
 36    """
 37
 38    def __init__(
 39        self,
 40        venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm',
 41        init_if_not_exists: bool = True,
 42        debug: bool = False,
 43    ) -> None:
 44        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
 45        ### For some weird threading issue,
 46        ### we can't use `isinstance` here.
 47        if '_Plugin' in str(type(venv)):
 48            self._venv = venv.name
 49            self._activate = venv.activate_venv
 50            self._deactivate = venv.deactivate_venv
 51            self._kwargs = {}
 52        else:
 53            self._venv = venv
 54            self._activate = activate_venv
 55            self._deactivate = deactivate_venv
 56            self._kwargs = {'venv': venv}
 57        self._debug = debug
 58        self._init_if_not_exists = init_if_not_exists
 59        ### In case someone calls `deactivate()` before `activate()`.
 60        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 61
 62
 63    def activate(self, debug: bool = False) -> bool:
 64        """
 65        Activate this virtual environment.
 66        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 67        will also be activated.
 68        """
 69        from meerschaum.utils.venv import active_venvs, init_venv
 70        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 71        try:
 72            return self._activate(
 73                debug=(debug or self._debug),
 74                init_if_not_exists=self._init_if_not_exists,
 75                **self._kwargs
 76            )
 77        except OSError as e:
 78            if self._init_if_not_exists:
 79                if not init_venv(self._venv, force=True):
 80                    raise e
 81        return self._activate(
 82            debug=(debug or self._debug),
 83            init_if_not_exists=self._init_if_not_exists,
 84            **self._kwargs
 85        )
 86
 87
 88    def deactivate(self, debug: bool = False) -> bool:
 89        """
 90        Deactivate this virtual environment.
 91        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 92        will also be deactivated.
 93        """
 94        return self._deactivate(debug=(debug or self._debug), **self._kwargs)
 95
 96
 97    @property
 98    def target_path(self) -> pathlib.Path:
 99        """
100        Return the target site-packages path for this virtual environment.
101        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
102        (e.g. Python 3.10 and Python 3.7).
103        """
104        from meerschaum.utils.venv import venv_target_path
105        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
106
107
108    @property
109    def root_path(self) -> pathlib.Path:
110        """
111        Return the top-level path for this virtual environment.
112        """
113        import meerschaum.config.paths as paths
114        if self._venv is None:
115            return self.target_path.parent
116        return paths.VIRTENV_RESOURCES_PATH / self._venv
117
118
119    def __enter__(self) -> None:
120        self.activate(debug=self._debug)
121
122
123    def __exit__(self, exc_type, exc_value, exc_traceback) -> None:
124        self.deactivate(debug=self._debug)
125
126
127    def __str__(self) -> str:
128        quote = "'" if self._venv is not None else ""
129        return "Venv(" + quote + str(self._venv) + quote + ")"
130
131
132    def __repr__(self) -> str:
133        return self.__str__()

Manage a virtual enviroment's activation status.

Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
...     import pandas
>>> with Venv(Plugin('noaa')) as venv:
...     import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
Venv( venv: Union[str, meerschaum.Plugin, NoneType] = 'mrsm', init_if_not_exists: bool = True, debug: bool = False)
38    def __init__(
39        self,
40        venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm',
41        init_if_not_exists: bool = True,
42        debug: bool = False,
43    ) -> None:
44        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
45        ### For some weird threading issue,
46        ### we can't use `isinstance` here.
47        if '_Plugin' in str(type(venv)):
48            self._venv = venv.name
49            self._activate = venv.activate_venv
50            self._deactivate = venv.deactivate_venv
51            self._kwargs = {}
52        else:
53            self._venv = venv
54            self._activate = activate_venv
55            self._deactivate = deactivate_venv
56            self._kwargs = {'venv': venv}
57        self._debug = debug
58        self._init_if_not_exists = init_if_not_exists
59        ### In case someone calls `deactivate()` before `activate()`.
60        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
def activate(self, debug: bool = False) -> bool:
63    def activate(self, debug: bool = False) -> bool:
64        """
65        Activate this virtual environment.
66        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
67        will also be activated.
68        """
69        from meerschaum.utils.venv import active_venvs, init_venv
70        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
71        try:
72            return self._activate(
73                debug=(debug or self._debug),
74                init_if_not_exists=self._init_if_not_exists,
75                **self._kwargs
76            )
77        except OSError as e:
78            if self._init_if_not_exists:
79                if not init_venv(self._venv, force=True):
80                    raise e
81        return self._activate(
82            debug=(debug or self._debug),
83            init_if_not_exists=self._init_if_not_exists,
84            **self._kwargs
85        )

Activate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be activated.

def deactivate(self, debug: bool = False) -> bool:
88    def deactivate(self, debug: bool = False) -> bool:
89        """
90        Deactivate this virtual environment.
91        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
92        will also be deactivated.
93        """
94        return self._deactivate(debug=(debug or self._debug), **self._kwargs)

Deactivate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be deactivated.

target_path: pathlib.Path
 97    @property
 98    def target_path(self) -> pathlib.Path:
 99        """
100        Return the target site-packages path for this virtual environment.
101        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
102        (e.g. Python 3.10 and Python 3.7).
103        """
104        from meerschaum.utils.venv import venv_target_path
105        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)

Return the target site-packages path for this virtual environment. A meerschaum.utils.venv.Venv may have one virtual environment per minor Python version (e.g. Python 3.10 and Python 3.7).

root_path: pathlib.Path
108    @property
109    def root_path(self) -> pathlib.Path:
110        """
111        Return the top-level path for this virtual environment.
112        """
113        import meerschaum.config.paths as paths
114        if self._venv is None:
115            return self.target_path.parent
116        return paths.VIRTENV_RESOURCES_PATH / self._venv

Return the top-level path for this virtual environment.

def activate_venv( venv: Optional[str] = 'mrsm', color: bool = True, force: bool = False, init_if_not_exists: bool = True, debug: bool = False, **kw) -> bool:
 39def activate_venv(
 40    venv: Optional[str] = 'mrsm',
 41    color: bool = True,
 42    force: bool = False,
 43    init_if_not_exists: bool = True,
 44    debug: bool = False,
 45    **kw
 46) -> bool:
 47    """
 48    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 49
 50    Parameters
 51    ----------
 52    venv: Optional[str], default 'mrsm'
 53        The virtual environment to activate.
 54
 55    color: bool, default True
 56        If `True`, include color in debug text.
 57
 58    init_if_not_exists: bool, default True
 59        If `True`, create the virtual environment if it does not exist.
 60
 61    force: bool, default False
 62        If `True`, do not exit early even if the venv is currently active.
 63
 64    debug: bool, default False
 65        Verbosity toggle.
 66
 67    Returns
 68    -------
 69    A bool indicating whether the virtual environment was successfully activated.
 70
 71    """
 72    thread_id = get_ident()
 73    if active_venvs_order and active_venvs_order[0] == venv:
 74        if not force:
 75            return True
 76
 77    import sys
 78    import os
 79    if venv is not None and init_if_not_exists:
 80        init_venv(venv=venv, debug=debug)
 81
 82    with LOCKS['active_venvs']:
 83        if thread_id not in threads_active_venvs:
 84            threads_active_venvs[thread_id] = {}
 85        active_venvs.add(venv)
 86        if venv not in threads_active_venvs[thread_id]:
 87            threads_active_venvs[thread_id][venv] = 1
 88        else:
 89            threads_active_venvs[thread_id][venv] += 1
 90
 91        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 92        if not target_path.exists() and venv is not None and init_if_not_exists:
 93            init_venv(venv=venv, force=True, debug=debug)
 94
 95        if not target_path.exists():
 96            if init_if_not_exists:
 97                return False
 98            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 99
100        target = target_path.as_posix()
101
102        if venv in active_venvs_order:
103            try:
104                sys.path.remove(target)
105            except Exception:
106                pass
107            try:
108                active_venvs_order.remove(venv)
109            except Exception:
110                pass
111
112        if venv is not None:
113            sys.path.insert(0, target)
114        else:
115            if sys.path and sys.path[0] in (os.getcwd(), ''):
116                sys.path.insert(1, target)
117            else:
118                sys.path.insert(0, target)
119        try:
120            active_venvs_order.insert(0, venv)
121        except Exception:
122            pass
123
124    return True

Create a virtual environment (if it doesn't exist) and add it to sys.path if necessary.

Parameters
  • venv (Optional[str], default 'mrsm'): The virtual environment to activate.
  • color (bool, default True): If True, include color in debug text.
  • init_if_not_exists (bool, default True): If True, create the virtual environment if it does not exist.
  • force (bool, default False): If True, do not exit early even if the venv is currently active.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment was successfully activated.
def deactivate_venv( venv: str = 'mrsm', color: bool = True, debug: bool = False, previously_active_venvs: Union[set[str], List[str], NoneType] = None, force: bool = False, **kw) -> bool:
127def deactivate_venv(
128    venv: str = 'mrsm',
129    color: bool = True,
130    debug: bool = False,
131    previously_active_venvs: Union['set[str]', List[str], None] = None,
132    force: bool = False,
133    **kw
134) -> bool:
135    """
136    Remove a virtual environment from `sys.path` (if it's been activated).
137
138    Parameters
139    ----------
140    venv: str, default 'mrsm'
141        The virtual environment to deactivate.
142
143    color: bool, default True
144        If `True`, include color in debug text.
145
146    debug: bool, default False
147        Verbosity toggle.
148
149    previously_active_venvs: Union[Set[str], List[str], None]
150        If provided, skip deactivating if a virtual environment is in this iterable.
151
152    force: bool, default False
153        If `True`, forcibly deactivate the virtual environment.
154        This may cause issues with other threads, so be careful!
155
156    Returns
157    -------
158    Return a bool indicating whether the virtual environment was successfully deactivated.
159
160    """
161    import sys
162    thread_id = get_ident()
163    if venv is None:
164        if venv in active_venvs:
165            active_venvs.remove(venv)
166        return True
167
168    if previously_active_venvs and venv in previously_active_venvs and not force:
169        return True
170
171    with LOCKS['active_venvs']:
172        if venv in threads_active_venvs.get(thread_id, {}):
173            new_count = threads_active_venvs[thread_id][venv] - 1
174            if new_count > 0 and not force:
175                threads_active_venvs[thread_id][venv] = new_count
176                return True
177            else:
178                del threads_active_venvs[thread_id][venv]
179
180        if not force:
181            for other_thread_id, other_venvs in threads_active_venvs.items():
182                if other_thread_id == thread_id:
183                    continue
184                if venv in other_venvs:
185                    return True
186        else:
187            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
188            for other_thread_id in to_delete:
189                del threads_active_venvs[other_thread_id]
190
191        if venv in active_venvs:
192            active_venvs.remove(venv)
193
194    if sys.path is None:
195        return False
196
197    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
198    with LOCKS['sys.path']:
199        if target in sys.path:
200            try:
201                sys.path.remove(target)
202            except Exception:
203                pass
204            try:
205                active_venvs_order.remove(venv)
206            except Exception:
207                pass
208
209    return True

Remove a virtual environment from sys.path (if it's been activated).

Parameters
  • venv (str, default 'mrsm'): The virtual environment to deactivate.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
  • previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
  • force (bool, default False): If True, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
  • Return a bool indicating whether the virtual environment was successfully deactivated.
def get_module_venv(module) -> Optional[str]:
849def get_module_venv(module) -> Union[str, None]:
850    """
851    Return the virtual environment where an imported module is installed.
852
853    Parameters
854    ----------
855    module: ModuleType
856        The imported module to inspect.
857
858    Returns
859    -------
860    The name of a venv or `None`.
861    """
862    import meerschaum.config.paths as paths
863    module_path = pathlib.Path(module.__file__).resolve()
864    try:
865        rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH)
866    except ValueError:
867        return None
868
869    return rel_path.as_posix().split('/', maxsplit=1)[0]

Return the virtual environment where an imported module is installed.

Parameters
  • module (ModuleType): The imported module to inspect.
Returns
  • The name of a venv or None.
def get_venvs() -> List[str]:
832def get_venvs() -> List[str]:
833    """
834    Return a list of all the virtual environments.
835    """
836    import os
837    import meerschaum.config.paths as paths
838    venvs = []
839    for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH):
840        path = paths.VIRTENV_RESOURCES_PATH / filename
841        if not path.is_dir():
842            continue
843        if not venv_exists(filename):
844            continue
845        venvs.append(filename)
846    return venvs

Return a list of all the virtual environments.

def init_venv( venv: str = 'mrsm', verify: bool = True, force: bool = False, debug: bool = False) -> bool:
368def init_venv(
369    venv: str = 'mrsm',
370    verify: bool = True,
371    force: bool = False,
372    debug: bool = False,
373) -> bool:
374    """
375    Initialize the virtual environment.
376
377    Parameters
378    ----------
379    venv: str, default 'mrsm'
380        The name of the virtual environment to create.
381
382    verify: bool, default True
383        If `True`, verify that the virtual environment is in the expected state.
384
385    force: bool, default False
386        If `True`, recreate the virtual environment, even if already initalized.
387
388    Returns
389    -------
390    A `bool` indicating success.
391    """
392    if not force and venv in verified_venvs:
393        return True
394    if not force and venv_exists(venv, debug=debug):
395        if verify:
396            verify_venv(venv, debug=debug)
397            verified_venvs.add(venv)
398        return True
399
400    import io
401    from contextlib import redirect_stdout
402    import sys
403    import platform
404    import os
405    import shutil
406    import time
407
408    import meerschaum.config.paths as paths
409    from meerschaum._internal.static import STATIC_CONFIG
410    from meerschaum.utils.packages import is_uv_enabled
411
412    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
413    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
414    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
415    lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
416    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
417
418    def update_lock(active: bool):
419        try:
420            if not active:
421                if debug:
422                    print(f"Releasing lock: '{lock_path}'")
423                lock_path.unlink()
424            else:
425                if debug:
426                    print(f"Acquiring lock: '{lock_path}'")
427                lock_path.touch()
428        except Exception:
429            pass
430
431    def wait_for_lock():
432        if platform.system() == 'Windows':
433            return
434        max_lock_seconds = 30.0
435        sleep_message_seconds = 5.0
436        step_sleep_seconds = 0.1
437        init_venv_check_start = time.perf_counter()
438        last_print = init_venv_check_start
439        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
440            if not lock_path.exists():
441                break
442
443            now = time.perf_counter()
444            if debug or (now - last_print) > sleep_message_seconds:
445                print(f"Lock exists for venv '{venv}', sleeping...")
446                last_print = now
447            time.sleep(step_sleep_seconds)
448        update_lock(False)
449
450    if (
451        not force
452        and venv == 'mrsm'
453        and os.environ.get(work_dir_env_var, None) is not None
454        and docker_home_venv_path.exists()
455    ):
456        wait_for_lock()
457        shutil.move(docker_home_venv_path, venv_path)
458        if verify:
459            verify_venv(venv, debug=debug)
460            verified_venvs.add(venv)
461        return True
462
463    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
464    global tried_virtualenv
465    try:
466        import venv as _venv
467        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
468        virtualenv = None
469    except ImportError:
470        _venv = None
471        uv = None
472        virtualenv = None
473
474    _venv_success = False
475    temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv)
476    ### NOTE: Disable site-packages movement for now.
477    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
478
479    wait_for_lock()
480    update_lock(True)
481
482    if rename_vtp:
483        if debug:
484            print(f"Moving '{vtp}' to '{temp_vtp}'...")
485        shutil.move(vtp, temp_vtp)
486
487    if uv is not None:
488        _venv_success = run_python_package(
489            'uv',
490            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
491            venv=None,
492            env=_get_pip_os_env(),
493            debug=debug,
494        ) == 0
495
496    if _venv is not None and not _venv_success:
497        f = io.StringIO()
498        with redirect_stdout(f):
499            _venv_success = run_python_package(
500                'venv',
501                [venv_path.as_posix()] + (
502                    ['--symlinks']
503                    if platform.system() != 'Windows'
504                    else []
505                ),
506                venv=None, debug=debug
507            ) == 0
508        if not _venv_success:
509            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
510        if not venv_exists(venv, debug=debug):
511            _venv = None
512    if not _venv_success:
513        virtualenv = attempt_import(
514            'virtualenv',
515            venv=None,
516            lazy=False,
517            install=(not tried_virtualenv),
518            warn=False,
519            check_update=False,
520            color=False,
521            debug=debug,
522        )
523        if virtualenv is None:
524            print(
525                "Failed to import `venv` or `virtualenv`! "
526                + "Please install `virtualenv` via pip then restart Meerschaum."
527            )
528            if rename_vtp and temp_vtp.exists():
529                if debug:
530                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
531                shutil.move(temp_vtp, vtp)
532            update_lock(False)
533            return False
534
535        tried_virtualenv = True
536        try:
537            python_folder = (
538                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
539            )
540            dist_packages_path = (
541                paths.VIRTENV_RESOURCES_PATH /
542                venv / 'local' / 'lib' / python_folder / 'dist-packages'
543            )
544            local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
545            bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin'
546            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
547            if bin_path.exists():
548                try:
549                    shutil.rmtree(bin_path)
550                except Exception:
551                    import traceback
552                    traceback.print_exc()
553            virtualenv.cli_run([venv_path.as_posix()])
554            if dist_packages_path.exists():
555                vtp.mkdir(exist_ok=True, parents=True)
556                for file_path in dist_packages_path.glob('*'):
557                    shutil.move(file_path, vtp)
558                shutil.rmtree(dist_packages_path)
559                #  shutil.move(dist_packages_path, vtp)
560                bin_path.mkdir(exist_ok=True, parents=True)
561                for file_path in local_bin_path.glob('*'):
562                    shutil.move(file_path, bin_path)
563                #  shutil.move(local_bin_path, bin_path)
564                shutil.rmtree(local_bin_path)
565
566        except Exception:
567            import traceback
568            traceback.print_exc()
569            if rename_vtp and temp_vtp.exists():
570                shutil.move(temp_vtp, vtp)
571            update_lock(False)
572            return False
573    if verify:
574        verify_venv(venv, debug=debug)
575        verified_venvs.add(venv)
576
577    if rename_vtp and temp_vtp.exists():
578        if debug:
579            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
580        shutil.move(temp_vtp, vtp)
581
582    update_lock(False)
583    return True

Initialize the virtual environment.

Parameters
  • venv (str, default 'mrsm'): The name of the virtual environment to create.
  • verify (bool, default True): If True, verify that the virtual environment is in the expected state.
  • force (bool, default False): If True, recreate the virtual environment, even if already initalized.
Returns
  • A bool indicating success.
def inside_venv() -> bool:
819def inside_venv() -> bool:
820    """
821    Determine whether current Python interpreter is running inside a virtual environment.
822    """
823    import sys
824    return (
825        hasattr(sys, 'real_prefix') or (
826            hasattr(sys, 'base_prefix')
827                and sys.base_prefix != sys.prefix
828        )
829    )

Determine whether current Python interpreter is running inside a virtual environment.

def is_venv_active(venv: str = 'mrsm', color: bool = True, debug: bool = False) -> bool:
212def is_venv_active(
213    venv: str = 'mrsm',
214    color : bool = True,
215    debug: bool = False
216) -> bool:
217    """
218    Check if a virtual environment is active.
219
220    Parameters
221    ----------
222    venv: str, default 'mrsm'
223        The virtual environment to check.
224
225    color: bool, default True
226        If `True`, include color in debug text.
227
228    debug: bool, default False
229        Verbosity toggle.
230
231    Returns
232    -------
233    A bool indicating whether the virtual environment `venv` is active.
234
235    """
236    return venv in active_venvs

Check if a virtual environment is active.

Parameters
  • venv (str, default 'mrsm'): The virtual environment to check.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment venv is active.
def venv_exec( code: str, venv: Optional[str] = 'mrsm', env: Optional[Dict[str, str]] = None, with_extras: bool = False, as_proc: bool = False, capture_output: bool = True, debug: bool = False) -> Union[bool, Tuple[int, bytes, bytes], subprocess.Popen]:
611def venv_exec(
612    code: str,
613    venv: Optional[str] = 'mrsm',
614    env: Optional[Dict[str, str]] = None,
615    with_extras: bool = False,
616    as_proc: bool = False,
617    capture_output: bool = True,
618    debug: bool = False,
619) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
620    """
621    Execute Python code in a subprocess via a virtual environment's interpeter.
622    Return `True` if the code successfully executes, `False` on failure.
623
624    Parameters
625    ----------
626    code: str
627        The Python code to excecute.
628
629    venv: str, default 'mrsm'
630        The virtual environment to use to get the path for the Python executable.
631        If `venv` is `None`, use the default `sys.executable` path.
632
633    env: Optional[Dict[str, str]], default None
634        Optionally specify the environment variables for the subprocess.
635        Defaults to `os.environ`.
636
637    with_extras: bool, default False
638        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
639
640    as_proc: bool, default False
641        If `True`, return the `subprocess.Popen` object instead of executing.
642
643    Returns
644    -------
645    By default, return a bool indicating success.
646    If `as_proc` is `True`, return a `subprocess.Popen` object.
647    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
648
649    """
650    import os
651    import subprocess
652    import platform
653    from meerschaum.utils.debug import dprint
654    from meerschaum.utils.process import _child_processes
655
656    executable = venv_executable(venv=venv)
657    cmd_list = [executable, '-c', code]
658    if env is None:
659        env = os.environ
660    if debug:
661        dprint(str(cmd_list))
662    if not with_extras and not as_proc:
663        return subprocess.call(cmd_list, env=env) == 0
664
665    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
666    group_kwargs = (
667        {
668            'preexec_fn': os.setsid,
669        } if platform.system() != 'Windows'
670        else {
671            'creationflags': CREATE_NEW_PROCESS_GROUP,
672        }
673    )
674    process = subprocess.Popen(
675        cmd_list,
676        stdout=stdout,
677        stderr=stderr,
678        stdin=sys.stdin,
679        env=env,
680        **group_kwargs
681    )
682    if as_proc:
683        _child_processes.append(process)
684        return process
685    stdout, stderr = process.communicate()
686    exit_code = process.returncode
687    return exit_code, stdout, stderr

Execute Python code in a subprocess via a virtual environment's interpeter. Return True if the code successfully executes, False on failure.

Parameters
  • code (str): The Python code to excecute.
  • venv (str, default 'mrsm'): The virtual environment to use to get the path for the Python executable. If venv is None, use the default sys.executable path.
  • env (Optional[Dict[str, str]], default None): Optionally specify the environment variables for the subprocess. Defaults to os.environ.
  • with_extras (bool, default False): If True, return a tuple of the exit code, stdout bytes, and stderr bytes.
  • as_proc (bool, default False): If True, return the subprocess.Popen object instead of executing.
Returns
  • By default, return a bool indicating success.
  • If as_proc is True, return a subprocess.Popen object.
  • If with_extras is True, return a tuple of the exit code, stdout bytes, and stderr bytes.
def venv_executable(venv: Optional[str] = 'mrsm') -> str:
586def venv_executable(venv: Optional[str] = 'mrsm') -> str:
587    """
588    The Python interpreter executable for a given virtual environment.
589    """
590    import sys
591    import platform
592    import meerschaum.config.paths as paths
593    return (
594        sys.executable if venv is None
595        else str(
596            paths.VIRTENV_RESOURCES_PATH
597            / venv
598            / (
599                'bin' if platform.system() != 'Windows'
600                else 'Scripts'
601            ) / (
602                'python'
603                + str(sys.version_info.major)
604                + '.'
605                + str(sys.version_info.minor)
606            )
607        )
608    )

The Python interpreter executable for a given virtual environment.

def venv_exists(venv: Optional[str], debug: bool = False) -> bool:
690def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
691    """
692    Determine whether a virtual environment has been created.
693    """
694    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
695    return target_path.exists()

Determine whether a virtual environment has been created.

def venv_target_path( venv: Optional[str], allow_nonexistent: bool = False, debug: bool = False) -> pathlib.Path:
698def venv_target_path(
699    venv: Union[str, None],
700    allow_nonexistent: bool = False,
701    debug: bool = False,
702) -> pathlib.Path:
703    """
704    Return a virtual environment's site-package path.
705
706    Parameters
707    ----------
708    venv: Union[str, None]
709        The virtual environment for which a path should be returned.
710
711    allow_nonexistent: bool, default False
712        If `True`, return a path even if it does not exist.
713
714    Returns
715    -------
716    The `pathlib.Path` object for the virtual environment's path.
717
718    """
719    import os
720    import sys
721    import platform
722    import site
723
724    import meerschaum.config.paths as paths
725    from meerschaum._internal.static import STATIC_CONFIG
726
727    ### Check sys.path for a user-writable site-packages directory.
728    if venv is None:
729
730        ### Return the known value for the portable environment.
731        environment_runtime = STATIC_CONFIG['environment']['runtime']
732        if os.environ.get(environment_runtime, None) == 'portable':
733            python_version_folder = (
734                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
735            )
736            executable_path = pathlib.Path(sys.executable)
737            site_packages_path = (
738                (
739                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
740                ) if platform.system() != 'Windows' else (
741                    executable_path.parent / 'Lib' / 'site-packages'
742                )
743            )
744            if not site_packages_path.exists():
745                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
746            return site_packages_path
747
748        if not inside_venv():
749            user_site_packages = site.getusersitepackages()
750            if user_site_packages is None:
751                raise EnvironmentError("Could not determine user site packages.")
752
753            site_path = pathlib.Path(user_site_packages)
754            if not site_path.exists():
755
756                ### Windows does not have `os.geteuid()`.
757                if platform.system() == 'Windows' or os.geteuid() != 0:
758                    site_path.mkdir(parents=True, exist_ok=True)
759                    return site_path
760
761                ### Allow for dist-level paths (running as root).
762                for possible_dist in site.getsitepackages():
763                    dist_path = pathlib.Path(possible_dist)
764                    if not dist_path.exists():
765                        continue
766                    return dist_path
767                
768                raise EnvironmentError("Could not determine the dist-packages directory.")
769
770            return site_path
771
772    venv_root_path = (
773        (paths.VIRTENV_RESOURCES_PATH / venv)
774        if venv is not None
775        else pathlib.Path(sys.prefix)
776    )
777    target_path = venv_root_path
778
779    ### Ensure 'lib' or 'Lib' exists.
780    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
781    if not allow_nonexistent:
782        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
783            print(f"Failed to find lib directory for virtual environment '{venv}'.")
784            import traceback
785            traceback.print_stack()
786            sys.exit(1)
787    target_path = target_path / lib
788
789    ### Check if a 'python3.x' folder exists.
790    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
791    if target_path.exists():
792        target_path = (
793            (target_path / python_folder)
794            if python_folder in os.listdir(target_path)
795            else target_path
796        )
797    else:
798        target_path = (
799            (target_path / python_folder)
800            if platform.system() != 'Windows'
801            else target_path
802        )
803
804    ### Ensure 'site-packages' exists.
805    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
806        target_path = target_path / 'site-packages'
807    else:
808        import traceback
809        traceback.print_stack()
810        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
811        print("This may be because you are using a different Python version.")
812        print("Try deleting the following directory and restarting Meerschaum:")
813        print(paths.VIRTENV_RESOURCES_PATH)
814        sys.exit(1)
815
816    return target_path

Return a virtual environment's site-package path.

Parameters
  • venv (Union[str, None]): The virtual environment for which a path should be returned.
  • allow_nonexistent (bool, default False): If True, return a path even if it does not exist.
Returns
  • The pathlib.Path object for the virtual environment's path.
def verify_venv(venv: str, debug: bool = False) -> None:
240def verify_venv(
241    venv: str,
242    debug: bool = False,
243) -> None:
244    """
245    Verify that the virtual environment matches the expected state.
246    """
247    import platform
248    import os
249    import shutil
250    import sys
251
252    import meerschaum.config.paths as paths
253    from meerschaum.utils.process import run_process
254    from meerschaum.utils.misc import make_symlink, is_symlink
255    from meerschaum.utils.warnings import warn
256
257    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
258    bin_path = venv_path / (
259        'bin' if platform.system() != 'Windows' else "Scripts"
260    )
261    current_python_versioned_name = (
262        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
263        + ('' if platform.system() != 'Windows' else '.exe')
264    )
265
266    if not (bin_path / current_python_versioned_name).exists():
267        init_venv(venv, verify=False, force=True, debug=debug)
268        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
269        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
270        if not current_python_in_venv_path.exists():
271            if is_symlink(current_python_in_venv_path):
272                try:
273                    current_python_in_venv_path.unlink()
274                except Exception as e:
275                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
276            try:
277                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
278            except Exception:
279                print(
280                    f"Unable to create symlink {current_python_in_venv_path} "
281                    + f"to {current_python_in_sys_path}."
282                )
283        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
284    else:
285        files_to_inspect = [current_python_versioned_name]
286
287    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
288        """
289        Return the version for the python binary at the given path.
290        """
291        try:
292            ### It might be a broken symlink, so skip on errors.
293            if debug:
294                print(f"Getting python version for {python_path}")
295            proc = run_process(
296                [str(python_path), '-V'],
297                as_proc=True,
298                capture_output=True,
299            )
300            stdout, stderr = proc.communicate(timeout=1.0)
301        except Exception as e:
302            ### E.g. the symlink may be broken.
303            if is_symlink(python_path):
304                try:
305                    python_path.unlink()
306                except Exception as _e:
307                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
308            return None
309        return stdout.decode('utf-8').strip().replace('Python ', '')
310
311    ### Ensure the versions are symlinked correctly.
312    for filename in files_to_inspect:
313        if not filename.startswith('python'):
314            continue
315        python_path = bin_path / filename
316        version = get_python_version(python_path)
317        if version is None:
318            continue
319        try:
320            major_version = version.split('.', maxsplit=1)[0]
321            minor_version = version.split('.', maxsplit=2)[1]
322        except IndexError:
323            return
324        python_versioned_name = (
325            'python' + major_version + '.' + minor_version
326            + ('' if platform.system() != 'Windows' else '.exe')
327        )
328
329        ### E.g. python3.10 actually links to Python 3.10.
330        if filename == python_versioned_name:
331            try:
332                real_path = pathlib.Path(os.path.realpath(python_path))
333                real_path_exists = real_path.exists()
334            except Exception:
335                real_path_exists = False
336
337            if not real_path_exists:
338                try:
339                    python_path.unlink()
340                except Exception:
341                    pass
342                init_venv(venv, verify=False, force=True, debug=debug)
343                if not python_path.exists():
344                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
345
346            if python_path == real_path:
347                continue
348
349            try:
350                python_path.unlink()
351            except Exception:
352                pass
353            success, msg = make_symlink(real_path, python_path)
354            if not success:
355                warn(msg, color=False)
356            continue
357
358        python_versioned_path = bin_path / python_versioned_name
359        if python_versioned_path.exists():
360            ### Avoid circular symlinks.
361            if get_python_version(python_versioned_path) == version:
362                continue
363            python_versioned_path.unlink()
364        shutil.move(python_path, python_versioned_path)

Verify that the virtual environment matches the expected state.