meerschaum.utils.venv

Manage virtual environments.

  1#! /usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3# vim:fenc=utf-8
  4
  5"""
  6Manage virtual environments.
  7"""
  8
  9from __future__ import annotations
 10
 11import sys
 12import pathlib
 13
 14from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple
 15from meerschaum.utils.threading import RLock, get_ident
 16
 17__all__ = sorted([
 18    'activate_venv', 'deactivate_venv', 'init_venv',
 19    'inside_venv', 'is_venv_active', 'venv_exec',
 20    'venv_executable', 'venv_exists', 'venv_target_path',
 21    'Venv', 'get_venvs', 'verify_venv', 'get_module_venv',
 22])
 23__pdoc__ = {'Venv': True}
 24
 25LOCKS = {
 26    'sys.path': RLock(),
 27    'active_venvs': RLock(),
 28    'venvs_active_counts': RLock(),
 29}
 30
 31active_venvs = set()
 32active_venvs_counts: Dict[str, int] = {}
 33active_venvs_order: List[Optional[str]] = []
 34threads_active_venvs: Dict[int, 'set[str]'] = {}
 35CREATE_NEW_PROCESS_GROUP = 0x00000200
 36
 37
 38def activate_venv(
 39    venv: Optional[str] = 'mrsm',
 40    color: bool = True,
 41    force: bool = False,
 42    init_if_not_exists: bool = True,
 43    debug: bool = False,
 44    **kw
 45) -> bool:
 46    """
 47    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 48
 49    Parameters
 50    ----------
 51    venv: Optional[str], default 'mrsm'
 52        The virtual environment to activate.
 53
 54    color: bool, default True
 55        If `True`, include color in debug text.
 56
 57    init_if_not_exists: bool, default True
 58        If `True`, create the virtual environment if it does not exist.
 59
 60    force: bool, default False
 61        If `True`, do not exit early even if the venv is currently active.
 62
 63    debug: bool, default False
 64        Verbosity toggle.
 65
 66    Returns
 67    -------
 68    A bool indicating whether the virtual environment was successfully activated.
 69
 70    """
 71    thread_id = get_ident()
 72    if active_venvs_order and active_venvs_order[0] == venv:
 73        if not force:
 74            return True
 75
 76    import sys
 77    import os
 78    if venv is not None and init_if_not_exists:
 79        init_venv(venv=venv, debug=debug)
 80
 81    with LOCKS['active_venvs']:
 82        if thread_id not in threads_active_venvs:
 83            threads_active_venvs[thread_id] = {}
 84        active_venvs.add(venv)
 85        if venv not in threads_active_venvs[thread_id]:
 86            threads_active_venvs[thread_id][venv] = 1
 87        else:
 88            threads_active_venvs[thread_id][venv] += 1
 89
 90        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 91        if not target_path.exists() and venv is not None and init_if_not_exists:
 92            init_venv(venv=venv, force=True, debug=debug)
 93
 94        if not target_path.exists():
 95            if init_if_not_exists:
 96                return False
 97            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 98
 99        target = target_path.as_posix()
100
101        if venv in active_venvs_order:
102            try:
103                sys.path.remove(target)
104            except Exception:
105                pass
106            try:
107                active_venvs_order.remove(venv)
108            except Exception:
109                pass
110
111        if venv is not None:
112            sys.path.insert(0, target)
113        else:
114            if sys.path and sys.path[0] in (os.getcwd(), ''):
115                sys.path.insert(1, target)
116            else:
117                sys.path.insert(0, target)
118        try:
119            active_venvs_order.insert(0, venv)
120        except Exception:
121            pass
122
123    return True
124
125
126def deactivate_venv(
127    venv: str = 'mrsm',
128    color: bool = True,
129    debug: bool = False,
130    previously_active_venvs: Union['set[str]', List[str], None] = None,
131    force: bool = False,
132    **kw
133) -> bool:
134    """
135    Remove a virtual environment from `sys.path` (if it's been activated).
136
137    Parameters
138    ----------
139    venv: str, default 'mrsm'
140        The virtual environment to deactivate.
141
142    color: bool, default True
143        If `True`, include color in debug text.
144
145    debug: bool, default False
146        Verbosity toggle.
147
148    previously_active_venvs: Union[Set[str], List[str], None]
149        If provided, skip deactivating if a virtual environment is in this iterable.
150
151    force: bool, default False
152        If `True`, forcibly deactivate the virtual environment.
153        This may cause issues with other threads, so be careful!
154
155    Returns
156    -------
157    Return a bool indicating whether the virtual environment was successfully deactivated.
158
159    """
160    import sys
161    thread_id = get_ident()
162    if venv is None:
163        if venv in active_venvs:
164            active_venvs.remove(venv)
165        return True
166
167    if previously_active_venvs and venv in previously_active_venvs and not force:
168        return True
169
170    with LOCKS['active_venvs']:
171        if venv in threads_active_venvs.get(thread_id, {}):
172            new_count = threads_active_venvs[thread_id][venv] - 1
173            if new_count > 0 and not force:
174                threads_active_venvs[thread_id][venv] = new_count
175                return True
176            else:
177                del threads_active_venvs[thread_id][venv]
178
179        if not force:
180            for other_thread_id, other_venvs in threads_active_venvs.items():
181                if other_thread_id == thread_id:
182                    continue
183                if venv in other_venvs:
184                    return True
185        else:
186            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
187            for other_thread_id in to_delete:
188                del threads_active_venvs[other_thread_id]
189
190        if venv in active_venvs:
191            active_venvs.remove(venv)
192
193    if sys.path is None:
194        return False
195
196    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
197    with LOCKS['sys.path']:
198        if target in sys.path:
199            try:
200                sys.path.remove(target)
201            except Exception:
202                pass
203            try:
204                active_venvs_order.remove(venv)
205            except Exception:
206                pass
207
208    return True
209
210
211def is_venv_active(
212    venv: str = 'mrsm',
213    color : bool = True,
214    debug: bool = False
215) -> bool:
216    """
217    Check if a virtual environment is active.
218
219    Parameters
220    ----------
221    venv: str, default 'mrsm'
222        The virtual environment to check.
223
224    color: bool, default True
225        If `True`, include color in debug text.
226
227    debug: bool, default False
228        Verbosity toggle.
229
230    Returns
231    -------
232    A bool indicating whether the virtual environment `venv` is active.
233
234    """
235    return venv in active_venvs
236
237
238verified_venvs = set()
239def verify_venv(
240    venv: str,
241    debug: bool = False,
242) -> None:
243    """
244    Verify that the virtual environment matches the expected state.
245    """
246    import platform
247    import os
248    import shutil
249    import sys
250    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
251    from meerschaum.utils.process import run_process
252    from meerschaum.utils.misc import make_symlink, is_symlink
253    from meerschaum.utils.warnings import warn
254
255    venv_path = VIRTENV_RESOURCES_PATH / venv
256    bin_path = venv_path / (
257        'bin' if platform.system() != 'Windows' else "Scripts"
258    )
259    current_python_versioned_name = (
260        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
261        + ('' if platform.system() != 'Windows' else '.exe')
262    )
263
264    if not (bin_path / current_python_versioned_name).exists():
265        init_venv(venv, verify=False, force=True, debug=debug)
266        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
267        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
268        if not current_python_in_venv_path.exists():
269            if is_symlink(current_python_in_venv_path):
270                try:
271                    current_python_in_venv_path.unlink()
272                except Exception as e:
273                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
274            try:
275                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
276            except Exception:
277                print(
278                    f"Unable to create symlink {current_python_in_venv_path} "
279                    + f"to {current_python_in_sys_path}."
280                )
281        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
282    else:
283        files_to_inspect = [current_python_versioned_name]
284
285    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
286        """
287        Return the version for the python binary at the given path.
288        """
289        try:
290            ### It might be a broken symlink, so skip on errors.
291            if debug:
292                print(f"Getting python version for {python_path}")
293            proc = run_process(
294                [str(python_path), '-V'],
295                as_proc=True,
296                capture_output=True,
297            )
298            stdout, stderr = proc.communicate(timeout=1.0)
299        except Exception as e:
300            ### E.g. the symlink may be broken.
301            if is_symlink(python_path):
302                try:
303                    python_path.unlink()
304                except Exception as _e:
305                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
306            return None
307        return stdout.decode('utf-8').strip().replace('Python ', '')
308
309    ### Ensure the versions are symlinked correctly.
310    for filename in files_to_inspect:
311        if not filename.startswith('python'):
312            continue
313        python_path = bin_path / filename
314        version = get_python_version(python_path)
315        if version is None:
316            continue
317        try:
318            major_version = version.split('.', maxsplit=1)[0]
319            minor_version = version.split('.', maxsplit=2)[1]
320        except IndexError:
321            return
322        python_versioned_name = (
323            'python' + major_version + '.' + minor_version
324            + ('' if platform.system() != 'Windows' else '.exe')
325        )
326
327        ### E.g. python3.10 actually links to Python 3.10.
328        if filename == python_versioned_name:
329            try:
330                real_path = pathlib.Path(os.path.realpath(python_path))
331                real_path_exists = real_path.exists()
332            except Exception:
333                real_path_exists = False
334
335            if not real_path_exists:
336                try:
337                    python_path.unlink()
338                except Exception:
339                    pass
340                init_venv(venv, verify=False, force=True, debug=debug)
341                if not python_path.exists():
342                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
343
344            if python_path == real_path:
345                continue
346
347            try:
348                python_path.unlink()
349            except Exception:
350                pass
351            success, msg = make_symlink(real_path, python_path)
352            if not success:
353                warn(msg, color=False)
354            continue
355
356        python_versioned_path = bin_path / python_versioned_name
357        if python_versioned_path.exists():
358            ### Avoid circular symlinks.
359            if get_python_version(python_versioned_path) == version:
360                continue
361            python_versioned_path.unlink()
362        shutil.move(python_path, python_versioned_path)
363
364
365tried_virtualenv = False
366def init_venv(
367    venv: str = 'mrsm',
368    verify: bool = True,
369    force: bool = False,
370    debug: bool = False,
371) -> bool:
372    """
373    Initialize the virtual environment.
374
375    Parameters
376    ----------
377    venv: str, default 'mrsm'
378        The name of the virtual environment to create.
379
380    verify: bool, default True
381        If `True`, verify that the virtual environment is in the expected state.
382
383    force: bool, default False
384        If `True`, recreate the virtual environment, even if already initalized.
385
386    Returns
387    -------
388    A `bool` indicating success.
389    """
390    if not force and venv in verified_venvs:
391        return True
392    if not force and venv_exists(venv, debug=debug):
393        if verify:
394            verify_venv(venv, debug=debug)
395            verified_venvs.add(venv)
396        return True
397
398    import io
399    from contextlib import redirect_stdout
400    import sys
401    import platform
402    import os
403    import shutil
404    import time
405
406    from meerschaum._internal.static import STATIC_CONFIG
407    from meerschaum.config._paths import (
408        VIRTENV_RESOURCES_PATH,
409        VENVS_CACHE_RESOURCES_PATH,
410    )
411    from meerschaum.utils.packages import is_uv_enabled
412
413    venv_path = VIRTENV_RESOURCES_PATH / venv
414    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
415    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
416    lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
417    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
418
419    def update_lock(active: bool):
420        try:
421            if not active:
422                if debug:
423                    print(f"Releasing lock: '{lock_path}'")
424                lock_path.unlink()
425            else:
426                if debug:
427                    print(f"Acquiring lock: '{lock_path}'")
428                lock_path.touch()
429        except Exception:
430            pass
431
432    def wait_for_lock():
433        if platform.system() == 'Windows':
434            return
435        max_lock_seconds = 30.0
436        sleep_message_seconds = 5.0
437        step_sleep_seconds = 0.1
438        init_venv_check_start = time.perf_counter()
439        last_print = init_venv_check_start
440        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
441            if not lock_path.exists():
442                break
443
444            now = time.perf_counter()
445            if debug or (now - last_print) > sleep_message_seconds:
446                print(f"Lock exists for venv '{venv}', sleeping...")
447                last_print = now
448            time.sleep(step_sleep_seconds)
449        update_lock(False)
450
451    if (
452        not force
453        and venv == 'mrsm'
454        and os.environ.get(work_dir_env_var, None) is not None
455        and docker_home_venv_path.exists()
456    ):
457        wait_for_lock()
458        shutil.move(docker_home_venv_path, venv_path)
459        if verify:
460            verify_venv(venv, debug=debug)
461            verified_venvs.add(venv)
462        return True
463
464    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
465    global tried_virtualenv
466    try:
467        import venv as _venv
468        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
469        virtualenv = None
470    except ImportError:
471        _venv = None
472        uv = None
473        virtualenv = None
474
475    _venv_success = False
476    temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv)
477    ### NOTE: Disable site-packages movement for now.
478    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
479
480    wait_for_lock()
481    update_lock(True)
482
483    if rename_vtp:
484        if debug:
485            print(f"Moving '{vtp}' to '{temp_vtp}'...")
486        shutil.move(vtp, temp_vtp)
487
488    if uv is not None:
489        _venv_success = run_python_package(
490            'uv',
491            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
492            venv=None,
493            env=_get_pip_os_env(),
494            debug=debug,
495        ) == 0
496
497    if _venv is not None and not _venv_success:
498        f = io.StringIO()
499        with redirect_stdout(f):
500            _venv_success = run_python_package(
501                'venv',
502                [venv_path.as_posix()] + (
503                    ['--symlinks']
504                    if platform.system() != 'Windows'
505                    else []
506                ),
507                venv=None, debug=debug
508            ) == 0
509        if not _venv_success:
510            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
511        if not venv_exists(venv, debug=debug):
512            _venv = None
513    if not _venv_success:
514        virtualenv = attempt_import(
515            'virtualenv',
516            venv=None,
517            lazy=False,
518            install=(not tried_virtualenv),
519            warn=False,
520            check_update=False,
521            color=False,
522            debug=debug,
523        )
524        if virtualenv is None:
525            print(
526                "Failed to import `venv` or `virtualenv`! "
527                + "Please install `virtualenv` via pip then restart Meerschaum."
528            )
529            if rename_vtp and temp_vtp.exists():
530                if debug:
531                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
532                shutil.move(temp_vtp, vtp)
533            update_lock(False)
534            return False
535
536        tried_virtualenv = True
537        try:
538            python_folder = (
539                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
540            )
541            dist_packages_path = (
542                VIRTENV_RESOURCES_PATH /
543                venv / 'local' / 'lib' / python_folder / 'dist-packages'
544            )
545            local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
546            bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin'
547            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
548            if bin_path.exists():
549                try:
550                    shutil.rmtree(bin_path)
551                except Exception:
552                    import traceback
553                    traceback.print_exc()
554            virtualenv.cli_run([venv_path.as_posix()])
555            if dist_packages_path.exists():
556                vtp.mkdir(exist_ok=True, parents=True)
557                for file_path in dist_packages_path.glob('*'):
558                    shutil.move(file_path, vtp)
559                shutil.rmtree(dist_packages_path)
560                #  shutil.move(dist_packages_path, vtp)
561                bin_path.mkdir(exist_ok=True, parents=True)
562                for file_path in local_bin_path.glob('*'):
563                    shutil.move(file_path, bin_path)
564                #  shutil.move(local_bin_path, bin_path)
565                shutil.rmtree(local_bin_path)
566
567        except Exception:
568            import traceback
569            traceback.print_exc()
570            if rename_vtp and temp_vtp.exists():
571                shutil.move(temp_vtp, vtp)
572            update_lock(False)
573            return False
574    if verify:
575        verify_venv(venv, debug=debug)
576        verified_venvs.add(venv)
577
578    if rename_vtp and temp_vtp.exists():
579        if debug:
580            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
581        shutil.move(temp_vtp, vtp)
582
583    update_lock(False)
584    return True
585
586
587def venv_executable(venv: Optional[str] = 'mrsm') -> str:
588    """
589    The Python interpreter executable for a given virtual environment.
590    """
591    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
592    import sys
593    import platform
594    return (
595        sys.executable if venv is None
596        else str(
597            VIRTENV_RESOURCES_PATH
598            / venv
599            / (
600                'bin' if platform.system() != 'Windows'
601                else 'Scripts'
602            ) / (
603                'python'
604                + str(sys.version_info.major)
605                + '.'
606                + str(sys.version_info.minor)
607            )
608        )
609    )
610
611
612def venv_exec(
613    code: str,
614    venv: Optional[str] = 'mrsm',
615    env: Optional[Dict[str, str]] = None,
616    with_extras: bool = False,
617    as_proc: bool = False,
618    capture_output: bool = True,
619    debug: bool = False,
620) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
621    """
622    Execute Python code in a subprocess via a virtual environment's interpeter.
623    Return `True` if the code successfully executes, `False` on failure.
624
625    Parameters
626    ----------
627    code: str
628        The Python code to excecute.
629
630    venv: str, default 'mrsm'
631        The virtual environment to use to get the path for the Python executable.
632        If `venv` is `None`, use the default `sys.executable` path.
633
634    env: Optional[Dict[str, str]], default None
635        Optionally specify the environment variables for the subprocess.
636        Defaults to `os.environ`.
637
638    with_extras: bool, default False
639        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
640
641    as_proc: bool, default False
642        If `True`, return the `subprocess.Popen` object instead of executing.
643
644    Returns
645    -------
646    By default, return a bool indicating success.
647    If `as_proc` is `True`, return a `subprocess.Popen` object.
648    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
649
650    """
651    import os
652    import subprocess
653    import platform
654    from meerschaum.utils.debug import dprint
655    from meerschaum.utils.process import _child_processes
656
657    executable = venv_executable(venv=venv)
658    cmd_list = [executable, '-c', code]
659    if env is None:
660        env = os.environ
661    if debug:
662        dprint(str(cmd_list))
663    if not with_extras and not as_proc:
664        return subprocess.call(cmd_list, env=env) == 0
665
666    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
667    group_kwargs = (
668        {
669            'preexec_fn': os.setsid,
670        } if platform.system() != 'Windows'
671        else {
672            'creationflags': CREATE_NEW_PROCESS_GROUP,
673        }
674    )
675    process = subprocess.Popen(
676        cmd_list,
677        stdout=stdout,
678        stderr=stderr,
679        stdin=sys.stdin,
680        env=env,
681        **group_kwargs
682    )
683    if as_proc:
684        _child_processes.append(process)
685        return process
686    stdout, stderr = process.communicate()
687    exit_code = process.returncode
688    return exit_code, stdout, stderr
689
690
691def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
692    """
693    Determine whether a virtual environment has been created.
694    """
695    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
696    return target_path.exists()
697
698
699def venv_target_path(
700    venv: Union[str, None],
701    allow_nonexistent: bool = False,
702    debug: bool = False,
703) -> pathlib.Path:
704    """
705    Return a virtual environment's site-package path.
706
707    Parameters
708    ----------
709    venv: Union[str, None]
710        The virtual environment for which a path should be returned.
711
712    allow_nonexistent: bool, default False
713        If `True`, return a path even if it does not exist.
714
715    Returns
716    -------
717    The `pathlib.Path` object for the virtual environment's path.
718
719    """
720    import os
721    import sys
722    import platform
723    import site
724    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
725    from meerschaum._internal.static import STATIC_CONFIG
726
727    ### Check sys.path for a user-writable site-packages directory.
728    if venv is None:
729
730        ### Return the known value for the portable environment.
731        environment_runtime = STATIC_CONFIG['environment']['runtime']
732        if os.environ.get(environment_runtime, None) == 'portable':
733            python_version_folder = (
734                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
735            )
736            executable_path = pathlib.Path(sys.executable)
737            site_packages_path = (
738                (
739                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
740                ) if platform.system() != 'Windows' else (
741                    executable_path.parent / 'Lib' / 'site-packages'
742                )
743            )
744            if not site_packages_path.exists():
745                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
746            return site_packages_path
747
748        if not inside_venv():
749            user_site_packages = site.getusersitepackages()
750            if user_site_packages is None:
751                raise EnvironmentError("Could not determine user site packages.")
752
753            site_path = pathlib.Path(user_site_packages)
754            if not site_path.exists():
755
756                ### Windows does not have `os.geteuid()`.
757                if platform.system() == 'Windows' or os.geteuid() != 0:
758                    site_path.mkdir(parents=True, exist_ok=True)
759                    return site_path
760
761                ### Allow for dist-level paths (running as root).
762                for possible_dist in site.getsitepackages():
763                    dist_path = pathlib.Path(possible_dist)
764                    if not dist_path.exists():
765                        continue
766                    return dist_path
767                
768                raise EnvironmentError("Could not determine the dist-packages directory.")
769
770            return site_path
771
772    venv_root_path = (
773        (VIRTENV_RESOURCES_PATH / venv)
774        if venv is not None
775        else pathlib.Path(sys.prefix)
776    )
777    target_path = venv_root_path
778
779    ### Ensure 'lib' or 'Lib' exists.
780    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
781    if not allow_nonexistent:
782        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
783            print(f"Failed to find lib directory for virtual environment '{venv}'.")
784            import traceback
785            traceback.print_stack()
786            sys.exit(1)
787    target_path = target_path / lib
788
789    ### Check if a 'python3.x' folder exists.
790    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
791    if target_path.exists():
792        target_path = (
793            (target_path / python_folder)
794            if python_folder in os.listdir(target_path)
795            else target_path
796        )
797    else:
798        target_path = (
799            (target_path / python_folder)
800            if platform.system() != 'Windows'
801            else target_path
802        )
803
804    ### Ensure 'site-packages' exists.
805    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
806        target_path = target_path / 'site-packages'
807    else:
808        import traceback
809        traceback.print_stack()
810        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
811        print("This may be because you are using a different Python version.")
812        print("Try deleting the following directory and restarting Meerschaum:")
813        print(VIRTENV_RESOURCES_PATH)
814        sys.exit(1)
815
816    return target_path
817
818
819def inside_venv() -> bool:
820    """
821    Determine whether current Python interpreter is running inside a virtual environment.
822    """
823    import sys
824    return (
825        hasattr(sys, 'real_prefix') or (
826            hasattr(sys, 'base_prefix')
827                and sys.base_prefix != sys.prefix
828        )
829    )
830
831
832def get_venvs() -> List[str]:
833    """
834    Return a list of all the virtual environments.
835    """
836    import os
837    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
838    venvs = []
839    for filename in os.listdir(VIRTENV_RESOURCES_PATH):
840        path = VIRTENV_RESOURCES_PATH / filename
841        if not path.is_dir():
842            continue
843        if not venv_exists(filename):
844            continue
845        venvs.append(filename)
846    return venvs
847
848
849def get_module_venv(module) -> Union[str, None]:
850    """
851    Return the virtual environment where an imported module is installed.
852
853    Parameters
854    ----------
855    module: ModuleType
856        The imported module to inspect.
857
858    Returns
859    -------
860    The name of a venv or `None`.
861    """
862    from meerschaum.config.paths import VIRTENV_RESOURCES_PATH
863    module_path = pathlib.Path(module.__file__).resolve()
864    try:
865        rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH)
866    except ValueError:
867        return None
868
869    return rel_path.as_posix().split('/', maxsplit=1)[0]
870
871
872from meerschaum.utils.venv._Venv import Venv
class Venv:
 19class Venv:
 20    """
 21    Manage a virtual enviroment's activation status.
 22
 23    Examples
 24    --------
 25    >>> from meerschaum.plugins import Plugin
 26    >>> with Venv('mrsm') as venv:
 27    ...     import pandas
 28    >>> with Venv(Plugin('noaa')) as venv:
 29    ...     import requests
 30    >>> venv = Venv('mrsm')
 31    >>> venv.activate()
 32    True
 33    >>> venv.deactivate()
 34    True
 35    >>> 
 36    """
 37
 38    def __init__(
 39        self,
 40        venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm',
 41        init_if_not_exists: bool = True,
 42        debug: bool = False,
 43    ) -> None:
 44        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
 45        ### For some weird threading issue,
 46        ### we can't use `isinstance` here.
 47        if '_Plugin' in str(type(venv)):
 48            self._venv = venv.name
 49            self._activate = venv.activate_venv
 50            self._deactivate = venv.deactivate_venv
 51            self._kwargs = {}
 52        else:
 53            self._venv = venv
 54            self._activate = activate_venv
 55            self._deactivate = deactivate_venv
 56            self._kwargs = {'venv': venv}
 57        self._debug = debug
 58        self._init_if_not_exists = init_if_not_exists
 59        ### In case someone calls `deactivate()` before `activate()`.
 60        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 61
 62
 63    def activate(self, debug: bool = False) -> bool:
 64        """
 65        Activate this virtual environment.
 66        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 67        will also be activated.
 68        """
 69        from meerschaum.utils.venv import active_venvs, init_venv
 70        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 71        try:
 72            return self._activate(
 73                debug=(debug or self._debug),
 74                init_if_not_exists=self._init_if_not_exists,
 75                **self._kwargs
 76            )
 77        except OSError as e:
 78            if self._init_if_not_exists:
 79                if not init_venv(self._venv, force=True):
 80                    raise e
 81        return self._activate(
 82            debug=(debug or self._debug),
 83            init_if_not_exists=self._init_if_not_exists,
 84            **self._kwargs
 85        )
 86
 87
 88    def deactivate(self, debug: bool = False) -> bool:
 89        """
 90        Deactivate this virtual environment.
 91        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 92        will also be deactivated.
 93        """
 94        return self._deactivate(debug=(debug or self._debug), **self._kwargs)
 95
 96
 97    @property
 98    def target_path(self) -> pathlib.Path:
 99        """
100        Return the target site-packages path for this virtual environment.
101        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
102        (e.g. Python 3.10 and Python 3.7).
103        """
104        from meerschaum.utils.venv import venv_target_path
105        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
106
107
108    @property
109    def root_path(self) -> pathlib.Path:
110        """
111        Return the top-level path for this virtual environment.
112        """
113        from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
114        if self._venv is None:
115            return self.target_path.parent
116        return VIRTENV_RESOURCES_PATH / self._venv
117
118
119    def __enter__(self) -> None:
120        self.activate(debug=self._debug)
121
122
123    def __exit__(self, exc_type, exc_value, exc_traceback) -> None:
124        self.deactivate(debug=self._debug)
125
126
127    def __str__(self) -> str:
128        quote = "'" if self._venv is not None else ""
129        return "Venv(" + quote + str(self._venv) + quote + ")"
130
131
132    def __repr__(self) -> str:
133        return self.__str__()

Manage a virtual enviroment's activation status.

Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
...     import pandas
>>> with Venv(Plugin('noaa')) as venv:
...     import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
Venv( venv: Union[str, meerschaum.Plugin, NoneType] = 'mrsm', init_if_not_exists: bool = True, debug: bool = False)
38    def __init__(
39        self,
40        venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm',
41        init_if_not_exists: bool = True,
42        debug: bool = False,
43    ) -> None:
44        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
45        ### For some weird threading issue,
46        ### we can't use `isinstance` here.
47        if '_Plugin' in str(type(venv)):
48            self._venv = venv.name
49            self._activate = venv.activate_venv
50            self._deactivate = venv.deactivate_venv
51            self._kwargs = {}
52        else:
53            self._venv = venv
54            self._activate = activate_venv
55            self._deactivate = deactivate_venv
56            self._kwargs = {'venv': venv}
57        self._debug = debug
58        self._init_if_not_exists = init_if_not_exists
59        ### In case someone calls `deactivate()` before `activate()`.
60        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
def activate(self, debug: bool = False) -> bool:
63    def activate(self, debug: bool = False) -> bool:
64        """
65        Activate this virtual environment.
66        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
67        will also be activated.
68        """
69        from meerschaum.utils.venv import active_venvs, init_venv
70        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
71        try:
72            return self._activate(
73                debug=(debug or self._debug),
74                init_if_not_exists=self._init_if_not_exists,
75                **self._kwargs
76            )
77        except OSError as e:
78            if self._init_if_not_exists:
79                if not init_venv(self._venv, force=True):
80                    raise e
81        return self._activate(
82            debug=(debug or self._debug),
83            init_if_not_exists=self._init_if_not_exists,
84            **self._kwargs
85        )

Activate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be activated.

def deactivate(self, debug: bool = False) -> bool:
88    def deactivate(self, debug: bool = False) -> bool:
89        """
90        Deactivate this virtual environment.
91        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
92        will also be deactivated.
93        """
94        return self._deactivate(debug=(debug or self._debug), **self._kwargs)

Deactivate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be deactivated.

target_path: pathlib.Path
 97    @property
 98    def target_path(self) -> pathlib.Path:
 99        """
100        Return the target site-packages path for this virtual environment.
101        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
102        (e.g. Python 3.10 and Python 3.7).
103        """
104        from meerschaum.utils.venv import venv_target_path
105        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)

Return the target site-packages path for this virtual environment. A meerschaum.utils.venv.Venv may have one virtual environment per minor Python version (e.g. Python 3.10 and Python 3.7).

root_path: pathlib.Path
108    @property
109    def root_path(self) -> pathlib.Path:
110        """
111        Return the top-level path for this virtual environment.
112        """
113        from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
114        if self._venv is None:
115            return self.target_path.parent
116        return VIRTENV_RESOURCES_PATH / self._venv

Return the top-level path for this virtual environment.

def activate_venv( venv: Optional[str] = 'mrsm', color: bool = True, force: bool = False, init_if_not_exists: bool = True, debug: bool = False, **kw) -> bool:
 39def activate_venv(
 40    venv: Optional[str] = 'mrsm',
 41    color: bool = True,
 42    force: bool = False,
 43    init_if_not_exists: bool = True,
 44    debug: bool = False,
 45    **kw
 46) -> bool:
 47    """
 48    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 49
 50    Parameters
 51    ----------
 52    venv: Optional[str], default 'mrsm'
 53        The virtual environment to activate.
 54
 55    color: bool, default True
 56        If `True`, include color in debug text.
 57
 58    init_if_not_exists: bool, default True
 59        If `True`, create the virtual environment if it does not exist.
 60
 61    force: bool, default False
 62        If `True`, do not exit early even if the venv is currently active.
 63
 64    debug: bool, default False
 65        Verbosity toggle.
 66
 67    Returns
 68    -------
 69    A bool indicating whether the virtual environment was successfully activated.
 70
 71    """
 72    thread_id = get_ident()
 73    if active_venvs_order and active_venvs_order[0] == venv:
 74        if not force:
 75            return True
 76
 77    import sys
 78    import os
 79    if venv is not None and init_if_not_exists:
 80        init_venv(venv=venv, debug=debug)
 81
 82    with LOCKS['active_venvs']:
 83        if thread_id not in threads_active_venvs:
 84            threads_active_venvs[thread_id] = {}
 85        active_venvs.add(venv)
 86        if venv not in threads_active_venvs[thread_id]:
 87            threads_active_venvs[thread_id][venv] = 1
 88        else:
 89            threads_active_venvs[thread_id][venv] += 1
 90
 91        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 92        if not target_path.exists() and venv is not None and init_if_not_exists:
 93            init_venv(venv=venv, force=True, debug=debug)
 94
 95        if not target_path.exists():
 96            if init_if_not_exists:
 97                return False
 98            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 99
100        target = target_path.as_posix()
101
102        if venv in active_venvs_order:
103            try:
104                sys.path.remove(target)
105            except Exception:
106                pass
107            try:
108                active_venvs_order.remove(venv)
109            except Exception:
110                pass
111
112        if venv is not None:
113            sys.path.insert(0, target)
114        else:
115            if sys.path and sys.path[0] in (os.getcwd(), ''):
116                sys.path.insert(1, target)
117            else:
118                sys.path.insert(0, target)
119        try:
120            active_venvs_order.insert(0, venv)
121        except Exception:
122            pass
123
124    return True

Create a virtual environment (if it doesn't exist) and add it to sys.path if necessary.

Parameters
  • venv (Optional[str], default 'mrsm'): The virtual environment to activate.
  • color (bool, default True): If True, include color in debug text.
  • init_if_not_exists (bool, default True): If True, create the virtual environment if it does not exist.
  • force (bool, default False): If True, do not exit early even if the venv is currently active.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment was successfully activated.
def deactivate_venv( venv: str = 'mrsm', color: bool = True, debug: bool = False, previously_active_venvs: Union[set[str], List[str], NoneType] = None, force: bool = False, **kw) -> bool:
127def deactivate_venv(
128    venv: str = 'mrsm',
129    color: bool = True,
130    debug: bool = False,
131    previously_active_venvs: Union['set[str]', List[str], None] = None,
132    force: bool = False,
133    **kw
134) -> bool:
135    """
136    Remove a virtual environment from `sys.path` (if it's been activated).
137
138    Parameters
139    ----------
140    venv: str, default 'mrsm'
141        The virtual environment to deactivate.
142
143    color: bool, default True
144        If `True`, include color in debug text.
145
146    debug: bool, default False
147        Verbosity toggle.
148
149    previously_active_venvs: Union[Set[str], List[str], None]
150        If provided, skip deactivating if a virtual environment is in this iterable.
151
152    force: bool, default False
153        If `True`, forcibly deactivate the virtual environment.
154        This may cause issues with other threads, so be careful!
155
156    Returns
157    -------
158    Return a bool indicating whether the virtual environment was successfully deactivated.
159
160    """
161    import sys
162    thread_id = get_ident()
163    if venv is None:
164        if venv in active_venvs:
165            active_venvs.remove(venv)
166        return True
167
168    if previously_active_venvs and venv in previously_active_venvs and not force:
169        return True
170
171    with LOCKS['active_venvs']:
172        if venv in threads_active_venvs.get(thread_id, {}):
173            new_count = threads_active_venvs[thread_id][venv] - 1
174            if new_count > 0 and not force:
175                threads_active_venvs[thread_id][venv] = new_count
176                return True
177            else:
178                del threads_active_venvs[thread_id][venv]
179
180        if not force:
181            for other_thread_id, other_venvs in threads_active_venvs.items():
182                if other_thread_id == thread_id:
183                    continue
184                if venv in other_venvs:
185                    return True
186        else:
187            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
188            for other_thread_id in to_delete:
189                del threads_active_venvs[other_thread_id]
190
191        if venv in active_venvs:
192            active_venvs.remove(venv)
193
194    if sys.path is None:
195        return False
196
197    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
198    with LOCKS['sys.path']:
199        if target in sys.path:
200            try:
201                sys.path.remove(target)
202            except Exception:
203                pass
204            try:
205                active_venvs_order.remove(venv)
206            except Exception:
207                pass
208
209    return True

Remove a virtual environment from sys.path (if it's been activated).

Parameters
  • venv (str, default 'mrsm'): The virtual environment to deactivate.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
  • previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
  • force (bool, default False): If True, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
  • Return a bool indicating whether the virtual environment was successfully deactivated.
def get_module_venv(module) -> Optional[str]:
850def get_module_venv(module) -> Union[str, None]:
851    """
852    Return the virtual environment where an imported module is installed.
853
854    Parameters
855    ----------
856    module: ModuleType
857        The imported module to inspect.
858
859    Returns
860    -------
861    The name of a venv or `None`.
862    """
863    from meerschaum.config.paths import VIRTENV_RESOURCES_PATH
864    module_path = pathlib.Path(module.__file__).resolve()
865    try:
866        rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH)
867    except ValueError:
868        return None
869
870    return rel_path.as_posix().split('/', maxsplit=1)[0]

Return the virtual environment where an imported module is installed.

Parameters
  • module (ModuleType): The imported module to inspect.
Returns
  • The name of a venv or None.
def get_venvs() -> List[str]:
833def get_venvs() -> List[str]:
834    """
835    Return a list of all the virtual environments.
836    """
837    import os
838    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
839    venvs = []
840    for filename in os.listdir(VIRTENV_RESOURCES_PATH):
841        path = VIRTENV_RESOURCES_PATH / filename
842        if not path.is_dir():
843            continue
844        if not venv_exists(filename):
845            continue
846        venvs.append(filename)
847    return venvs

Return a list of all the virtual environments.

def init_venv( venv: str = 'mrsm', verify: bool = True, force: bool = False, debug: bool = False) -> bool:
367def init_venv(
368    venv: str = 'mrsm',
369    verify: bool = True,
370    force: bool = False,
371    debug: bool = False,
372) -> bool:
373    """
374    Initialize the virtual environment.
375
376    Parameters
377    ----------
378    venv: str, default 'mrsm'
379        The name of the virtual environment to create.
380
381    verify: bool, default True
382        If `True`, verify that the virtual environment is in the expected state.
383
384    force: bool, default False
385        If `True`, recreate the virtual environment, even if already initalized.
386
387    Returns
388    -------
389    A `bool` indicating success.
390    """
391    if not force and venv in verified_venvs:
392        return True
393    if not force and venv_exists(venv, debug=debug):
394        if verify:
395            verify_venv(venv, debug=debug)
396            verified_venvs.add(venv)
397        return True
398
399    import io
400    from contextlib import redirect_stdout
401    import sys
402    import platform
403    import os
404    import shutil
405    import time
406
407    from meerschaum._internal.static import STATIC_CONFIG
408    from meerschaum.config._paths import (
409        VIRTENV_RESOURCES_PATH,
410        VENVS_CACHE_RESOURCES_PATH,
411    )
412    from meerschaum.utils.packages import is_uv_enabled
413
414    venv_path = VIRTENV_RESOURCES_PATH / venv
415    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
416    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
417    lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
418    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
419
420    def update_lock(active: bool):
421        try:
422            if not active:
423                if debug:
424                    print(f"Releasing lock: '{lock_path}'")
425                lock_path.unlink()
426            else:
427                if debug:
428                    print(f"Acquiring lock: '{lock_path}'")
429                lock_path.touch()
430        except Exception:
431            pass
432
433    def wait_for_lock():
434        if platform.system() == 'Windows':
435            return
436        max_lock_seconds = 30.0
437        sleep_message_seconds = 5.0
438        step_sleep_seconds = 0.1
439        init_venv_check_start = time.perf_counter()
440        last_print = init_venv_check_start
441        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
442            if not lock_path.exists():
443                break
444
445            now = time.perf_counter()
446            if debug or (now - last_print) > sleep_message_seconds:
447                print(f"Lock exists for venv '{venv}', sleeping...")
448                last_print = now
449            time.sleep(step_sleep_seconds)
450        update_lock(False)
451
452    if (
453        not force
454        and venv == 'mrsm'
455        and os.environ.get(work_dir_env_var, None) is not None
456        and docker_home_venv_path.exists()
457    ):
458        wait_for_lock()
459        shutil.move(docker_home_venv_path, venv_path)
460        if verify:
461            verify_venv(venv, debug=debug)
462            verified_venvs.add(venv)
463        return True
464
465    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
466    global tried_virtualenv
467    try:
468        import venv as _venv
469        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
470        virtualenv = None
471    except ImportError:
472        _venv = None
473        uv = None
474        virtualenv = None
475
476    _venv_success = False
477    temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv)
478    ### NOTE: Disable site-packages movement for now.
479    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
480
481    wait_for_lock()
482    update_lock(True)
483
484    if rename_vtp:
485        if debug:
486            print(f"Moving '{vtp}' to '{temp_vtp}'...")
487        shutil.move(vtp, temp_vtp)
488
489    if uv is not None:
490        _venv_success = run_python_package(
491            'uv',
492            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
493            venv=None,
494            env=_get_pip_os_env(),
495            debug=debug,
496        ) == 0
497
498    if _venv is not None and not _venv_success:
499        f = io.StringIO()
500        with redirect_stdout(f):
501            _venv_success = run_python_package(
502                'venv',
503                [venv_path.as_posix()] + (
504                    ['--symlinks']
505                    if platform.system() != 'Windows'
506                    else []
507                ),
508                venv=None, debug=debug
509            ) == 0
510        if not _venv_success:
511            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
512        if not venv_exists(venv, debug=debug):
513            _venv = None
514    if not _venv_success:
515        virtualenv = attempt_import(
516            'virtualenv',
517            venv=None,
518            lazy=False,
519            install=(not tried_virtualenv),
520            warn=False,
521            check_update=False,
522            color=False,
523            debug=debug,
524        )
525        if virtualenv is None:
526            print(
527                "Failed to import `venv` or `virtualenv`! "
528                + "Please install `virtualenv` via pip then restart Meerschaum."
529            )
530            if rename_vtp and temp_vtp.exists():
531                if debug:
532                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
533                shutil.move(temp_vtp, vtp)
534            update_lock(False)
535            return False
536
537        tried_virtualenv = True
538        try:
539            python_folder = (
540                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
541            )
542            dist_packages_path = (
543                VIRTENV_RESOURCES_PATH /
544                venv / 'local' / 'lib' / python_folder / 'dist-packages'
545            )
546            local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
547            bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin'
548            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
549            if bin_path.exists():
550                try:
551                    shutil.rmtree(bin_path)
552                except Exception:
553                    import traceback
554                    traceback.print_exc()
555            virtualenv.cli_run([venv_path.as_posix()])
556            if dist_packages_path.exists():
557                vtp.mkdir(exist_ok=True, parents=True)
558                for file_path in dist_packages_path.glob('*'):
559                    shutil.move(file_path, vtp)
560                shutil.rmtree(dist_packages_path)
561                #  shutil.move(dist_packages_path, vtp)
562                bin_path.mkdir(exist_ok=True, parents=True)
563                for file_path in local_bin_path.glob('*'):
564                    shutil.move(file_path, bin_path)
565                #  shutil.move(local_bin_path, bin_path)
566                shutil.rmtree(local_bin_path)
567
568        except Exception:
569            import traceback
570            traceback.print_exc()
571            if rename_vtp and temp_vtp.exists():
572                shutil.move(temp_vtp, vtp)
573            update_lock(False)
574            return False
575    if verify:
576        verify_venv(venv, debug=debug)
577        verified_venvs.add(venv)
578
579    if rename_vtp and temp_vtp.exists():
580        if debug:
581            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
582        shutil.move(temp_vtp, vtp)
583
584    update_lock(False)
585    return True

Initialize the virtual environment.

Parameters
  • venv (str, default 'mrsm'): The name of the virtual environment to create.
  • verify (bool, default True): If True, verify that the virtual environment is in the expected state.
  • force (bool, default False): If True, recreate the virtual environment, even if already initalized.
Returns
  • A bool indicating success.
def inside_venv() -> bool:
820def inside_venv() -> bool:
821    """
822    Determine whether current Python interpreter is running inside a virtual environment.
823    """
824    import sys
825    return (
826        hasattr(sys, 'real_prefix') or (
827            hasattr(sys, 'base_prefix')
828                and sys.base_prefix != sys.prefix
829        )
830    )

Determine whether current Python interpreter is running inside a virtual environment.

def is_venv_active(venv: str = 'mrsm', color: bool = True, debug: bool = False) -> bool:
212def is_venv_active(
213    venv: str = 'mrsm',
214    color : bool = True,
215    debug: bool = False
216) -> bool:
217    """
218    Check if a virtual environment is active.
219
220    Parameters
221    ----------
222    venv: str, default 'mrsm'
223        The virtual environment to check.
224
225    color: bool, default True
226        If `True`, include color in debug text.
227
228    debug: bool, default False
229        Verbosity toggle.
230
231    Returns
232    -------
233    A bool indicating whether the virtual environment `venv` is active.
234
235    """
236    return venv in active_venvs

Check if a virtual environment is active.

Parameters
  • venv (str, default 'mrsm'): The virtual environment to check.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment venv is active.
def venv_exec( code: str, venv: Optional[str] = 'mrsm', env: Optional[Dict[str, str]] = None, with_extras: bool = False, as_proc: bool = False, capture_output: bool = True, debug: bool = False) -> Union[bool, Tuple[int, bytes, bytes], subprocess.Popen]:
613def venv_exec(
614    code: str,
615    venv: Optional[str] = 'mrsm',
616    env: Optional[Dict[str, str]] = None,
617    with_extras: bool = False,
618    as_proc: bool = False,
619    capture_output: bool = True,
620    debug: bool = False,
621) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
622    """
623    Execute Python code in a subprocess via a virtual environment's interpeter.
624    Return `True` if the code successfully executes, `False` on failure.
625
626    Parameters
627    ----------
628    code: str
629        The Python code to excecute.
630
631    venv: str, default 'mrsm'
632        The virtual environment to use to get the path for the Python executable.
633        If `venv` is `None`, use the default `sys.executable` path.
634
635    env: Optional[Dict[str, str]], default None
636        Optionally specify the environment variables for the subprocess.
637        Defaults to `os.environ`.
638
639    with_extras: bool, default False
640        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
641
642    as_proc: bool, default False
643        If `True`, return the `subprocess.Popen` object instead of executing.
644
645    Returns
646    -------
647    By default, return a bool indicating success.
648    If `as_proc` is `True`, return a `subprocess.Popen` object.
649    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
650
651    """
652    import os
653    import subprocess
654    import platform
655    from meerschaum.utils.debug import dprint
656    from meerschaum.utils.process import _child_processes
657
658    executable = venv_executable(venv=venv)
659    cmd_list = [executable, '-c', code]
660    if env is None:
661        env = os.environ
662    if debug:
663        dprint(str(cmd_list))
664    if not with_extras and not as_proc:
665        return subprocess.call(cmd_list, env=env) == 0
666
667    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
668    group_kwargs = (
669        {
670            'preexec_fn': os.setsid,
671        } if platform.system() != 'Windows'
672        else {
673            'creationflags': CREATE_NEW_PROCESS_GROUP,
674        }
675    )
676    process = subprocess.Popen(
677        cmd_list,
678        stdout=stdout,
679        stderr=stderr,
680        stdin=sys.stdin,
681        env=env,
682        **group_kwargs
683    )
684    if as_proc:
685        _child_processes.append(process)
686        return process
687    stdout, stderr = process.communicate()
688    exit_code = process.returncode
689    return exit_code, stdout, stderr

Execute Python code in a subprocess via a virtual environment's interpeter. Return True if the code successfully executes, False on failure.

Parameters
  • code (str): The Python code to excecute.
  • venv (str, default 'mrsm'): The virtual environment to use to get the path for the Python executable. If venv is None, use the default sys.executable path.
  • env (Optional[Dict[str, str]], default None): Optionally specify the environment variables for the subprocess. Defaults to os.environ.
  • with_extras (bool, default False): If True, return a tuple of the exit code, stdout bytes, and stderr bytes.
  • as_proc (bool, default False): If True, return the subprocess.Popen object instead of executing.
Returns
  • By default, return a bool indicating success.
  • If as_proc is True, return a subprocess.Popen object.
  • If with_extras is True, return a tuple of the exit code, stdout bytes, and stderr bytes.
def venv_executable(venv: Optional[str] = 'mrsm') -> str:
588def venv_executable(venv: Optional[str] = 'mrsm') -> str:
589    """
590    The Python interpreter executable for a given virtual environment.
591    """
592    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
593    import sys
594    import platform
595    return (
596        sys.executable if venv is None
597        else str(
598            VIRTENV_RESOURCES_PATH
599            / venv
600            / (
601                'bin' if platform.system() != 'Windows'
602                else 'Scripts'
603            ) / (
604                'python'
605                + str(sys.version_info.major)
606                + '.'
607                + str(sys.version_info.minor)
608            )
609        )
610    )

The Python interpreter executable for a given virtual environment.

def venv_exists(venv: Optional[str], debug: bool = False) -> bool:
692def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
693    """
694    Determine whether a virtual environment has been created.
695    """
696    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
697    return target_path.exists()

Determine whether a virtual environment has been created.

def venv_target_path( venv: Optional[str], allow_nonexistent: bool = False, debug: bool = False) -> pathlib.Path:
700def venv_target_path(
701    venv: Union[str, None],
702    allow_nonexistent: bool = False,
703    debug: bool = False,
704) -> pathlib.Path:
705    """
706    Return a virtual environment's site-package path.
707
708    Parameters
709    ----------
710    venv: Union[str, None]
711        The virtual environment for which a path should be returned.
712
713    allow_nonexistent: bool, default False
714        If `True`, return a path even if it does not exist.
715
716    Returns
717    -------
718    The `pathlib.Path` object for the virtual environment's path.
719
720    """
721    import os
722    import sys
723    import platform
724    import site
725    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
726    from meerschaum._internal.static import STATIC_CONFIG
727
728    ### Check sys.path for a user-writable site-packages directory.
729    if venv is None:
730
731        ### Return the known value for the portable environment.
732        environment_runtime = STATIC_CONFIG['environment']['runtime']
733        if os.environ.get(environment_runtime, None) == 'portable':
734            python_version_folder = (
735                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
736            )
737            executable_path = pathlib.Path(sys.executable)
738            site_packages_path = (
739                (
740                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
741                ) if platform.system() != 'Windows' else (
742                    executable_path.parent / 'Lib' / 'site-packages'
743                )
744            )
745            if not site_packages_path.exists():
746                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
747            return site_packages_path
748
749        if not inside_venv():
750            user_site_packages = site.getusersitepackages()
751            if user_site_packages is None:
752                raise EnvironmentError("Could not determine user site packages.")
753
754            site_path = pathlib.Path(user_site_packages)
755            if not site_path.exists():
756
757                ### Windows does not have `os.geteuid()`.
758                if platform.system() == 'Windows' or os.geteuid() != 0:
759                    site_path.mkdir(parents=True, exist_ok=True)
760                    return site_path
761
762                ### Allow for dist-level paths (running as root).
763                for possible_dist in site.getsitepackages():
764                    dist_path = pathlib.Path(possible_dist)
765                    if not dist_path.exists():
766                        continue
767                    return dist_path
768                
769                raise EnvironmentError("Could not determine the dist-packages directory.")
770
771            return site_path
772
773    venv_root_path = (
774        (VIRTENV_RESOURCES_PATH / venv)
775        if venv is not None
776        else pathlib.Path(sys.prefix)
777    )
778    target_path = venv_root_path
779
780    ### Ensure 'lib' or 'Lib' exists.
781    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
782    if not allow_nonexistent:
783        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
784            print(f"Failed to find lib directory for virtual environment '{venv}'.")
785            import traceback
786            traceback.print_stack()
787            sys.exit(1)
788    target_path = target_path / lib
789
790    ### Check if a 'python3.x' folder exists.
791    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
792    if target_path.exists():
793        target_path = (
794            (target_path / python_folder)
795            if python_folder in os.listdir(target_path)
796            else target_path
797        )
798    else:
799        target_path = (
800            (target_path / python_folder)
801            if platform.system() != 'Windows'
802            else target_path
803        )
804
805    ### Ensure 'site-packages' exists.
806    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
807        target_path = target_path / 'site-packages'
808    else:
809        import traceback
810        traceback.print_stack()
811        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
812        print("This may be because you are using a different Python version.")
813        print("Try deleting the following directory and restarting Meerschaum:")
814        print(VIRTENV_RESOURCES_PATH)
815        sys.exit(1)
816
817    return target_path

Return a virtual environment's site-package path.

Parameters
  • venv (Union[str, None]): The virtual environment for which a path should be returned.
  • allow_nonexistent (bool, default False): If True, return a path even if it does not exist.
Returns
  • The pathlib.Path object for the virtual environment's path.
def verify_venv(venv: str, debug: bool = False) -> None:
240def verify_venv(
241    venv: str,
242    debug: bool = False,
243) -> None:
244    """
245    Verify that the virtual environment matches the expected state.
246    """
247    import platform
248    import os
249    import shutil
250    import sys
251    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
252    from meerschaum.utils.process import run_process
253    from meerschaum.utils.misc import make_symlink, is_symlink
254    from meerschaum.utils.warnings import warn
255
256    venv_path = VIRTENV_RESOURCES_PATH / venv
257    bin_path = venv_path / (
258        'bin' if platform.system() != 'Windows' else "Scripts"
259    )
260    current_python_versioned_name = (
261        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
262        + ('' if platform.system() != 'Windows' else '.exe')
263    )
264
265    if not (bin_path / current_python_versioned_name).exists():
266        init_venv(venv, verify=False, force=True, debug=debug)
267        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
268        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
269        if not current_python_in_venv_path.exists():
270            if is_symlink(current_python_in_venv_path):
271                try:
272                    current_python_in_venv_path.unlink()
273                except Exception as e:
274                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
275            try:
276                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
277            except Exception:
278                print(
279                    f"Unable to create symlink {current_python_in_venv_path} "
280                    + f"to {current_python_in_sys_path}."
281                )
282        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
283    else:
284        files_to_inspect = [current_python_versioned_name]
285
286    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
287        """
288        Return the version for the python binary at the given path.
289        """
290        try:
291            ### It might be a broken symlink, so skip on errors.
292            if debug:
293                print(f"Getting python version for {python_path}")
294            proc = run_process(
295                [str(python_path), '-V'],
296                as_proc=True,
297                capture_output=True,
298            )
299            stdout, stderr = proc.communicate(timeout=1.0)
300        except Exception as e:
301            ### E.g. the symlink may be broken.
302            if is_symlink(python_path):
303                try:
304                    python_path.unlink()
305                except Exception as _e:
306                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
307            return None
308        return stdout.decode('utf-8').strip().replace('Python ', '')
309
310    ### Ensure the versions are symlinked correctly.
311    for filename in files_to_inspect:
312        if not filename.startswith('python'):
313            continue
314        python_path = bin_path / filename
315        version = get_python_version(python_path)
316        if version is None:
317            continue
318        try:
319            major_version = version.split('.', maxsplit=1)[0]
320            minor_version = version.split('.', maxsplit=2)[1]
321        except IndexError:
322            return
323        python_versioned_name = (
324            'python' + major_version + '.' + minor_version
325            + ('' if platform.system() != 'Windows' else '.exe')
326        )
327
328        ### E.g. python3.10 actually links to Python 3.10.
329        if filename == python_versioned_name:
330            try:
331                real_path = pathlib.Path(os.path.realpath(python_path))
332                real_path_exists = real_path.exists()
333            except Exception:
334                real_path_exists = False
335
336            if not real_path_exists:
337                try:
338                    python_path.unlink()
339                except Exception:
340                    pass
341                init_venv(venv, verify=False, force=True, debug=debug)
342                if not python_path.exists():
343                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
344
345            if python_path == real_path:
346                continue
347
348            try:
349                python_path.unlink()
350            except Exception:
351                pass
352            success, msg = make_symlink(real_path, python_path)
353            if not success:
354                warn(msg, color=False)
355            continue
356
357        python_versioned_path = bin_path / python_versioned_name
358        if python_versioned_path.exists():
359            ### Avoid circular symlinks.
360            if get_python_version(python_versioned_path) == version:
361                continue
362            python_versioned_path.unlink()
363        shutil.move(python_path, python_versioned_path)

Verify that the virtual environment matches the expected state.