meerschaum.utils.venv

Manage virtual environments.

  1#! /usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3# vim:fenc=utf-8
  4
  5"""
  6Manage virtual environments.
  7"""
  8
  9from __future__ import annotations
 10
 11from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple
 12from meerschaum.utils.threading import RLock, get_ident
 13
 14__all__ = sorted([
 15    'activate_venv', 'deactivate_venv', 'init_venv',
 16    'inside_venv', 'is_venv_active', 'venv_exec',
 17    'venv_executable', 'venv_exists', 'venv_target_path',
 18    'Venv', 'get_venvs', 'verify_venv', 'get_module_venv',
 19])
 20__pdoc__ = {'Venv': True}
 21
 22LOCKS = {
 23    'sys.path': RLock(),
 24    'active_venvs': RLock(),
 25    'venvs_active_counts': RLock(),
 26}
 27
 28active_venvs = set()
 29active_venvs_counts: Dict[str, int] = {}
 30active_venvs_order: List[Optional[str]] = []
 31threads_active_venvs: Dict[int, 'set[str]'] = {}
 32CREATE_NEW_PROCESS_GROUP = 0x00000200
 33
 34
 35def activate_venv(
 36    venv: Optional[str] = 'mrsm',
 37    color: bool = True,
 38    force: bool = False,
 39    debug: bool = False,
 40    **kw
 41) -> bool:
 42    """
 43    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 44
 45    Parameters
 46    ----------
 47    venv: Optional[str], default 'mrsm'
 48        The virtual environment to activate.
 49
 50    color: bool, default True
 51        If `True`, include color in debug text.
 52
 53    force: bool, default False
 54        If `True`, do not exit early even if the venv is currently active.
 55
 56    debug: bool, default False
 57        Verbosity toggle.
 58
 59    Returns
 60    -------
 61    A bool indicating whether the virtual environment was successfully activated.
 62
 63    """
 64    thread_id = get_ident()
 65    if active_venvs_order and active_venvs_order[0] == venv:
 66        if not force:
 67            return True
 68    import sys
 69    import os
 70    if venv is not None:
 71        init_venv(venv=venv, debug=debug)
 72    with LOCKS['active_venvs']:
 73        if thread_id not in threads_active_venvs:
 74            threads_active_venvs[thread_id] = {}
 75        active_venvs.add(venv)
 76        if venv not in threads_active_venvs[thread_id]:
 77            threads_active_venvs[thread_id][venv] = 1
 78        else:
 79            threads_active_venvs[thread_id][venv] += 1
 80
 81        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 82        if not target_path.exists():
 83            init_venv(venv=venv, force=True, debug=debug)
 84        if not target_path.exists():
 85            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 86        target = target_path.as_posix()
 87
 88        if venv in active_venvs_order:
 89            try:
 90                sys.path.remove(target)
 91            except Exception:
 92                pass
 93            try:
 94                active_venvs_order.remove(venv)
 95            except Exception:
 96                pass
 97        if venv is not None:
 98            sys.path.insert(0, target)
 99        else:
100            if sys.path and sys.path[0] in (os.getcwd(), ''):
101                sys.path.insert(1, target)
102            else:
103                sys.path.insert(0, target)
104        try:
105            active_venvs_order.insert(0, venv)
106        except Exception:
107            pass
108
109    return True
110
111
112def deactivate_venv(
113    venv: str = 'mrsm',
114    color: bool = True,
115    debug: bool = False,
116    previously_active_venvs: Union['set[str]', List[str], None] = None,
117    force: bool = False,
118    **kw
119) -> bool:
120    """
121    Remove a virtual environment from `sys.path` (if it's been activated).
122
123    Parameters
124    ----------
125    venv: str, default 'mrsm'
126        The virtual environment to deactivate.
127
128    color: bool, default True
129        If `True`, include color in debug text.
130
131    debug: bool, default False
132        Verbosity toggle.
133
134    previously_active_venvs: Union[Set[str], List[str], None]
135        If provided, skip deactivating if a virtual environment is in this iterable.
136
137    force: bool, default False
138        If `True`, forcibly deactivate the virtual environment.
139        This may cause issues with other threads, so be careful!
140
141    Returns
142    -------
143    Return a bool indicating whether the virtual environment was successfully deactivated.
144
145    """
146    import sys
147    thread_id = get_ident()
148    if venv is None:
149        if venv in active_venvs:
150            active_venvs.remove(venv)
151        return True
152
153    if previously_active_venvs and venv in previously_active_venvs and not force:
154        return True
155
156    with LOCKS['active_venvs']:
157        if venv in threads_active_venvs.get(thread_id, {}):
158            new_count = threads_active_venvs[thread_id][venv] - 1
159            if new_count > 0 and not force:
160                threads_active_venvs[thread_id][venv] = new_count
161                return True
162            else:
163                del threads_active_venvs[thread_id][venv]
164
165        if not force:
166            for other_thread_id, other_venvs in threads_active_venvs.items():
167                if other_thread_id == thread_id:
168                    continue
169                if venv in other_venvs:
170                    return True
171        else:
172            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
173            for other_thread_id in to_delete:
174                del threads_active_venvs[other_thread_id]
175
176        if venv in active_venvs:
177            active_venvs.remove(venv)
178
179    if sys.path is None:
180        return False
181
182    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
183    with LOCKS['sys.path']:
184        if target in sys.path:
185            try:
186                sys.path.remove(target)
187            except Exception:
188                pass
189            try:
190                active_venvs_order.remove(venv)
191            except Exception:
192                pass
193
194    return True
195
196
197def is_venv_active(
198    venv: str = 'mrsm',
199    color : bool = True,
200    debug: bool = False
201) -> bool:
202    """
203    Check if a virtual environment is active.
204
205    Parameters
206    ----------
207    venv: str, default 'mrsm'
208        The virtual environment to check.
209
210    color: bool, default True
211        If `True`, include color in debug text.
212
213    debug: bool, default False
214        Verbosity toggle.
215
216    Returns
217    -------
218    A bool indicating whether the virtual environment `venv` is active.
219
220    """
221    return venv in active_venvs
222
223
224verified_venvs = set()
225def verify_venv(
226    venv: str,
227    debug: bool = False,
228) -> None:
229    """
230    Verify that the virtual environment matches the expected state.
231    """
232    import pathlib
233    import platform
234    import os
235    import shutil
236    import sys
237    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
238    from meerschaum.utils.process import run_process
239    from meerschaum.utils.misc import make_symlink, is_symlink
240    from meerschaum.utils.warnings import warn
241
242    venv_path = VIRTENV_RESOURCES_PATH / venv
243    bin_path = venv_path / (
244        'bin' if platform.system() != 'Windows' else "Scripts"
245    )
246    current_python_versioned_name = (
247        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
248        + ('' if platform.system() != 'Windows' else '.exe')
249    )
250
251    if not (bin_path / current_python_versioned_name).exists():
252        init_venv(venv, verify=False, force=True, debug=debug)
253        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
254        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
255        if not current_python_in_venv_path.exists():
256            if is_symlink(current_python_in_venv_path):
257                try:
258                    current_python_in_venv_path.unlink()
259                except Exception as e:
260                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
261            try:
262                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
263            except Exception:
264                print(
265                    f"Unable to create symlink {current_python_in_venv_path} "
266                    + f"to {current_python_in_sys_path}."
267                )
268        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
269    else:
270        files_to_inspect = [current_python_versioned_name]
271
272    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
273        """
274        Return the version for the python binary at the given path.
275        """
276        try:
277            ### It might be a broken symlink, so skip on errors.
278            if debug:
279                print(f"Getting python version for {python_path}")
280            proc = run_process(
281                [str(python_path), '-V'],
282                as_proc=True,
283                capture_output=True,
284            )
285            stdout, stderr = proc.communicate(timeout=1.0)
286        except Exception as e:
287            ### E.g. the symlink may be broken.
288            if is_symlink(python_path):
289                try:
290                    python_path.unlink()
291                except Exception as _e:
292                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
293            return None
294        return stdout.decode('utf-8').strip().replace('Python ', '')
295
296    ### Ensure the versions are symlinked correctly.
297    for filename in files_to_inspect:
298        if not filename.startswith('python'):
299            continue
300        python_path = bin_path / filename
301        version = get_python_version(python_path)
302        if version is None:
303            continue
304        try:
305            major_version = version.split('.', maxsplit=1)[0]
306            minor_version = version.split('.', maxsplit=2)[1]
307        except IndexError:
308            return
309        python_versioned_name = (
310            'python' + major_version + '.' + minor_version
311            + ('' if platform.system() != 'Windows' else '.exe')
312        )
313
314        ### E.g. python3.10 actually links to Python 3.10.
315        if filename == python_versioned_name:
316            real_path = pathlib.Path(os.path.realpath(python_path))
317            if not real_path.exists():
318                try:
319                    python_path.unlink()
320                except Exception:
321                    pass
322                init_venv(venv, verify=False, force=True, debug=debug)
323                if not python_path.exists():
324                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
325
326            if python_path == real_path:
327                continue
328
329            try:
330                python_path.unlink()
331            except Exception:
332                pass
333            success, msg = make_symlink(real_path, python_path)
334            if not success:
335                warn(msg, color=False)
336            continue
337
338        python_versioned_path = bin_path / python_versioned_name
339        if python_versioned_path.exists():
340            ### Avoid circular symlinks.
341            if get_python_version(python_versioned_path) == version:
342                continue
343            python_versioned_path.unlink()
344        shutil.move(python_path, python_versioned_path)
345
346
347tried_virtualenv = False
348def init_venv(
349    venv: str = 'mrsm',
350    verify: bool = True,
351    force: bool = False,
352    debug: bool = False,
353) -> bool:
354    """
355    Initialize the virtual environment.
356
357    Parameters
358    ----------
359    venv: str, default 'mrsm'
360        The name of the virtual environment to create.
361
362    verify: bool, default True
363        If `True`, verify that the virtual environment is in the expected state.
364
365    force: bool, default False
366        If `True`, recreate the virtual environment, even if already initalized.
367
368    Returns
369    -------
370    A `bool` indicating success.
371    """
372    if not force and venv in verified_venvs:
373        return True
374    if not force and venv_exists(venv, debug=debug):
375        if verify:
376            verify_venv(venv, debug=debug)
377            verified_venvs.add(venv)
378        return True
379
380    import io
381    from contextlib import redirect_stdout
382    import sys
383    import platform
384    import os
385    import pathlib
386    import shutil
387    import time
388
389    from meerschaum.config.static import STATIC_CONFIG
390    from meerschaum.config._paths import (
391        VIRTENV_RESOURCES_PATH,
392        VENVS_CACHE_RESOURCES_PATH,
393    )
394    from meerschaum.utils.packages import is_uv_enabled
395
396    venv_path = VIRTENV_RESOURCES_PATH / venv
397    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
398    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
399    lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
400    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
401
402    def update_lock(active: bool):
403        try:
404            if not active:
405                if debug:
406                    print(f"Releasing lock: '{lock_path}'")
407                lock_path.unlink()
408            else:
409                if debug:
410                    print(f"Acquiring lock: '{lock_path}'")
411                lock_path.touch()
412        except Exception:
413            pass
414
415    def wait_for_lock():
416        if platform.system() == 'Windows':
417            return
418        max_lock_seconds = 30.0
419        sleep_message_seconds = 5.0
420        step_sleep_seconds = 0.1
421        init_venv_check_start = time.perf_counter()
422        last_print = init_venv_check_start
423        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
424            if not lock_path.exists():
425                break
426
427            now = time.perf_counter()
428            if debug or (now - last_print) > sleep_message_seconds:
429                print(f"Lock exists for venv '{venv}', sleeping...")
430                last_print = now
431            time.sleep(step_sleep_seconds)
432        update_lock(False)
433
434    if (
435        not force
436        and venv == 'mrsm'
437        and os.environ.get(work_dir_env_var, None) is not None
438        and docker_home_venv_path.exists()
439    ):
440        wait_for_lock()
441        shutil.move(docker_home_venv_path, venv_path)
442        if verify:
443            verify_venv(venv, debug=debug)
444            verified_venvs.add(venv)
445        return True
446
447    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
448    global tried_virtualenv
449    try:
450        import venv as _venv
451        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
452        virtualenv = None
453    except ImportError:
454        _venv = None
455        uv = None
456        virtualenv = None
457
458    _venv_success = False
459    temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv)
460    ### NOTE: Disable site-packages movement for now.
461    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
462
463    wait_for_lock()
464    update_lock(True)
465
466    if rename_vtp:
467        if debug:
468            print(f"Moving '{vtp}' to '{temp_vtp}'...")
469        shutil.move(vtp, temp_vtp)
470
471    if uv is not None:
472        _venv_success = run_python_package(
473            'uv',
474            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
475            venv=None,
476            env=_get_pip_os_env(),
477            debug=debug,
478        ) == 0
479
480    if _venv is not None and not _venv_success:
481        f = io.StringIO()
482        with redirect_stdout(f):
483            _venv_success = run_python_package(
484                'venv',
485                [venv_path.as_posix()] + (
486                    ['--symlinks']
487                    if platform.system() != 'Windows'
488                    else []
489                ),
490                venv=None, debug=debug
491            ) == 0
492        if not _venv_success:
493            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
494        if not venv_exists(venv, debug=debug):
495            _venv = None
496    if not _venv_success:
497        virtualenv = attempt_import(
498            'virtualenv',
499            venv=None,
500            lazy=False,
501            install=(not tried_virtualenv),
502            warn=False,
503            check_update=False,
504            color=False,
505            debug=debug,
506        )
507        if virtualenv is None:
508            print(
509                "Failed to import `venv` or `virtualenv`! "
510                + "Please install `virtualenv` via pip then restart Meerschaum."
511            )
512            if rename_vtp and temp_vtp.exists():
513                if debug:
514                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
515                shutil.move(temp_vtp, vtp)
516            update_lock(False)
517            return False
518
519        tried_virtualenv = True
520        try:
521            python_folder = (
522                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
523            )
524            dist_packages_path = (
525                VIRTENV_RESOURCES_PATH /
526                venv / 'local' / 'lib' / python_folder / 'dist-packages'
527            )
528            local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
529            bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin'
530            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
531            if bin_path.exists():
532                try:
533                    shutil.rmtree(bin_path)
534                except Exception:
535                    import traceback
536                    traceback.print_exc()
537            virtualenv.cli_run([venv_path.as_posix()])
538            if dist_packages_path.exists():
539                vtp.mkdir(exist_ok=True, parents=True)
540                for file_path in dist_packages_path.glob('*'):
541                    shutil.move(file_path, vtp)
542                shutil.rmtree(dist_packages_path)
543                #  shutil.move(dist_packages_path, vtp)
544                bin_path.mkdir(exist_ok=True, parents=True)
545                for file_path in local_bin_path.glob('*'):
546                    shutil.move(file_path, bin_path)
547                #  shutil.move(local_bin_path, bin_path)
548                shutil.rmtree(local_bin_path)
549
550        except Exception:
551            import traceback
552            traceback.print_exc()
553            if rename_vtp and temp_vtp.exists():
554                shutil.move(temp_vtp, vtp)
555            update_lock(False)
556            return False
557    if verify:
558        verify_venv(venv, debug=debug)
559        verified_venvs.add(venv)
560
561    if rename_vtp and temp_vtp.exists():
562        if debug:
563            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
564        shutil.move(temp_vtp, vtp)
565
566    update_lock(False)
567    return True
568
569
570def venv_executable(venv: Optional[str] = 'mrsm') -> str:
571    """
572    The Python interpreter executable for a given virtual environment.
573    """
574    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
575    import sys
576    import platform
577    return (
578        sys.executable if venv is None
579        else str(
580            VIRTENV_RESOURCES_PATH
581            / venv
582            / (
583                'bin' if platform.system() != 'Windows'
584                else 'Scripts'
585            ) / (
586                'python'
587                + str(sys.version_info.major)
588                + '.'
589                + str(sys.version_info.minor)
590            )
591        )
592    )
593
594
595def venv_exec(
596    code: str,
597    venv: Optional[str] = 'mrsm',
598    env: Optional[Dict[str, str]] = None,
599    with_extras: bool = False,
600    as_proc: bool = False,
601    capture_output: bool = True,
602    debug: bool = False,
603) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
604    """
605    Execute Python code in a subprocess via a virtual environment's interpeter.
606    Return `True` if the code successfully executes, `False` on failure.
607
608    Parameters
609    ----------
610    code: str
611        The Python code to excecute.
612
613    venv: str, default 'mrsm'
614        The virtual environment to use to get the path for the Python executable.
615        If `venv` is `None`, use the default `sys.executable` path.
616
617    env: Optional[Dict[str, str]], default None
618        Optionally specify the environment variables for the subprocess.
619        Defaults to `os.environ`.
620
621    with_extras: bool, default False
622        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
623
624    as_proc: bool, default False
625        If `True`, return the `subprocess.Popen` object instead of executing.
626
627    Returns
628    -------
629    By default, return a bool indicating success.
630    If `as_proc` is `True`, return a `subprocess.Popen` object.
631    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
632
633    """
634    import os
635    import subprocess
636    import platform
637    from meerschaum.utils.debug import dprint
638    from meerschaum.utils.process import _child_processes
639
640    executable = venv_executable(venv=venv)
641    cmd_list = [executable, '-c', code]
642    if env is None:
643        env = os.environ
644    if debug:
645        dprint(str(cmd_list))
646    if not with_extras and not as_proc:
647        return subprocess.call(cmd_list, env=env) == 0
648
649    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
650    group_kwargs = (
651        {
652            'preexec_fn': os.setsid,
653        } if platform.system() != 'Windows'
654        else {
655            'creationflags': CREATE_NEW_PROCESS_GROUP,
656        }
657    )
658    process = subprocess.Popen(
659        cmd_list,
660        stdout=stdout,
661        stderr=stderr,
662        env=env,
663        **group_kwargs
664    )
665    if as_proc:
666        _child_processes.append(process)
667        return process
668    stdout, stderr = process.communicate()
669    exit_code = process.returncode
670    return exit_code, stdout, stderr
671
672
673def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
674    """
675    Determine whether a virtual environment has been created.
676    """
677    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
678    return target_path.exists()
679
680
681def venv_target_path(
682    venv: Union[str, None],
683    allow_nonexistent: bool = False,
684    debug: bool = False,
685) -> 'pathlib.Path':
686    """
687    Return a virtual environment's site-package path.
688
689    Parameters
690    ----------
691    venv: Union[str, None]
692        The virtual environment for which a path should be returned.
693
694    allow_nonexistent: bool, default False
695        If `True`, return a path even if it does not exist.
696
697    Returns
698    -------
699    The `pathlib.Path` object for the virtual environment's path.
700
701    """
702    import os
703    import sys
704    import platform
705    import pathlib
706    import site
707    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
708    from meerschaum.config.static import STATIC_CONFIG
709
710    ### Check sys.path for a user-writable site-packages directory.
711    if venv is None:
712
713        ### Return the known value for the portable environment.
714        environment_runtime = STATIC_CONFIG['environment']['runtime']
715        if os.environ.get(environment_runtime, None) == 'portable':
716            python_version_folder = (
717                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
718            )
719            executable_path = pathlib.Path(sys.executable)
720            site_packages_path = (
721                (
722                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
723                ) if platform.system() != 'Windows' else (
724                    executable_path.parent / 'Lib' / 'site-packages'
725                )
726            )
727            if not site_packages_path.exists():
728                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
729            return site_packages_path
730
731        if not inside_venv():
732            user_site_packages = site.getusersitepackages()
733            if user_site_packages is None:
734                raise EnvironmentError("Could not determine user site packages.")
735
736            site_path = pathlib.Path(user_site_packages)
737            if not site_path.exists():
738
739                ### Windows does not have `os.geteuid()`.
740                if platform.system() == 'Windows' or os.geteuid() != 0:
741                    site_path.mkdir(parents=True, exist_ok=True)
742                    return site_path
743
744                ### Allow for dist-level paths (running as root).
745                for possible_dist in site.getsitepackages():
746                    dist_path = pathlib.Path(possible_dist)
747                    if not dist_path.exists():
748                        continue
749                    return dist_path
750                
751                raise EnvironmentError("Could not determine the dist-packages directory.")
752
753            return site_path
754
755    venv_root_path = (
756        (VIRTENV_RESOURCES_PATH / venv)
757        if venv is not None else pathlib.Path(sys.prefix)
758    )
759    target_path = venv_root_path
760
761    ### Ensure 'lib' or 'Lib' exists.
762    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
763    if not allow_nonexistent:
764        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
765            print(f"Failed to find lib directory for virtual environment '{venv}'.")
766            import traceback
767            traceback.print_stack()
768            sys.exit(1)
769    target_path = target_path / lib
770
771    ### Check if a 'python3.x' folder exists.
772    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
773    if target_path.exists():
774        target_path = (
775            (target_path / python_folder) if python_folder in os.listdir(target_path)
776            else target_path
777        )
778    else:
779        target_path = (
780            (target_path / python_folder) if platform.system() != 'Windows'
781            else target_path
782        )
783
784    ### Ensure 'site-packages' exists.
785    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
786        target_path = target_path / 'site-packages'
787    else:
788        import traceback
789        traceback.print_stack()
790        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
791        print("This may be because you are using a different Python version.")
792        print("Try deleting the following directory and restarting Meerschaum:")
793        print(VIRTENV_RESOURCES_PATH)
794        sys.exit(1)
795
796    return target_path
797
798
799def inside_venv() -> bool:
800    """
801    Determine whether current Python interpreter is running inside a virtual environment.
802    """
803    import sys
804    return (
805        hasattr(sys, 'real_prefix') or (
806            hasattr(sys, 'base_prefix')
807                and sys.base_prefix != sys.prefix
808        )
809    )
810
811
812def get_venvs() -> List[str]:
813    """
814    Return a list of all the virtual environments.
815    """
816    import os
817    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
818    venvs = []
819    for filename in os.listdir(VIRTENV_RESOURCES_PATH):
820        path = VIRTENV_RESOURCES_PATH / filename
821        if not path.is_dir():
822            continue
823        if not venv_exists(filename):
824            continue
825        venvs.append(filename)
826    return venvs
827
828
829def get_module_venv(module) -> Union[str, None]:
830    """
831    Return the virtual environment where an imported module is installed.
832
833    Parameters
834    ----------
835    module: ModuleType
836        The imported module to inspect.
837
838    Returns
839    -------
840    The name of a venv or `None`.
841    """
842    import pathlib
843    from meerschaum.config.paths import VIRTENV_RESOURCES_PATH
844    module_path = pathlib.Path(module.__file__).resolve()
845    try:
846        rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH)
847    except ValueError:
848        return None
849
850    return rel_path.as_posix().split('/', maxsplit=1)[0]
851
852
853from meerschaum.utils.venv._Venv import Venv
class Venv:
 18class Venv:
 19    """
 20    Manage a virtual enviroment's activation status.
 21
 22    Examples
 23    --------
 24    >>> from meerschaum.plugins import Plugin
 25    >>> with Venv('mrsm') as venv:
 26    ...     import pandas
 27    >>> with Venv(Plugin('noaa')) as venv:
 28    ...     import requests
 29    >>> venv = Venv('mrsm')
 30    >>> venv.activate()
 31    True
 32    >>> venv.deactivate()
 33    True
 34    >>> 
 35    """
 36
 37    def __init__(
 38        self,
 39        venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
 40        debug: bool = False,
 41    ) -> None:
 42        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
 43        ### For some weird threading issue,
 44        ### we can't use `isinstance` here.
 45        if 'meerschaum.plugins._Plugin' in str(type(venv)):
 46            self._venv = venv.name
 47            self._activate = venv.activate_venv
 48            self._deactivate = venv.deactivate_venv
 49            self._kwargs = {}
 50        else:
 51            self._venv = venv
 52            self._activate = activate_venv
 53            self._deactivate = deactivate_venv
 54            self._kwargs = {'venv': venv}
 55        self._debug = debug
 56        ### In case someone calls `deactivate()` before `activate()`.
 57        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 58
 59
 60    def activate(self, debug: bool = False) -> bool:
 61        """
 62        Activate this virtual environment.
 63        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 64        will also be activated.
 65        """
 66        from meerschaum.utils.venv import active_venvs, init_venv
 67        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 68        try:
 69            return self._activate(debug=(debug or self._debug), **self._kwargs)
 70        except OSError as e:
 71            if not init_venv(self._venv, force=True):
 72                raise e
 73        return self._activate(debug=(debug or self._debug), **self._kwargs)
 74
 75
 76    def deactivate(self, debug: bool = False) -> bool:
 77        """
 78        Deactivate this virtual environment.
 79        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 80        will also be deactivated.
 81        """
 82        return self._deactivate(debug=(debug or self._debug), **self._kwargs)
 83
 84
 85    @property
 86    def target_path(self) -> pathlib.Path:
 87        """
 88        Return the target site-packages path for this virtual environment.
 89        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
 90        (e.g. Python 3.10 and Python 3.7).
 91        """
 92        from meerschaum.utils.venv import venv_target_path
 93        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
 94
 95
 96    @property
 97    def root_path(self) -> pathlib.Path:
 98        """
 99        Return the top-level path for this virtual environment.
100        """
101        from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
102        if self._venv is None:
103            return self.target_path.parent
104        return VIRTENV_RESOURCES_PATH / self._venv
105
106
107    def __enter__(self) -> None:
108        self.activate(debug=self._debug)
109
110
111    def __exit__(self, exc_type, exc_value, exc_traceback) -> None:
112        self.deactivate(debug=self._debug)
113
114
115    def __str__(self) -> str:
116        quote = "'" if self._venv is not None else ""
117        return "Venv(" + quote + str(self._venv) + quote + ")"
118
119
120    def __repr__(self) -> str:
121        return self.__str__()

Manage a virtual enviroment's activation status.

Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
...     import pandas
>>> with Venv(Plugin('noaa')) as venv:
...     import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
Venv( venv: Union[str, meerschaum.Plugin, NoneType] = 'mrsm', debug: bool = False)
37    def __init__(
38        self,
39        venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
40        debug: bool = False,
41    ) -> None:
42        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
43        ### For some weird threading issue,
44        ### we can't use `isinstance` here.
45        if 'meerschaum.plugins._Plugin' in str(type(venv)):
46            self._venv = venv.name
47            self._activate = venv.activate_venv
48            self._deactivate = venv.deactivate_venv
49            self._kwargs = {}
50        else:
51            self._venv = venv
52            self._activate = activate_venv
53            self._deactivate = deactivate_venv
54            self._kwargs = {'venv': venv}
55        self._debug = debug
56        ### In case someone calls `deactivate()` before `activate()`.
57        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
def activate(self, debug: bool = False) -> bool:
60    def activate(self, debug: bool = False) -> bool:
61        """
62        Activate this virtual environment.
63        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
64        will also be activated.
65        """
66        from meerschaum.utils.venv import active_venvs, init_venv
67        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
68        try:
69            return self._activate(debug=(debug or self._debug), **self._kwargs)
70        except OSError as e:
71            if not init_venv(self._venv, force=True):
72                raise e
73        return self._activate(debug=(debug or self._debug), **self._kwargs)

Activate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be activated.

def deactivate(self, debug: bool = False) -> bool:
76    def deactivate(self, debug: bool = False) -> bool:
77        """
78        Deactivate this virtual environment.
79        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
80        will also be deactivated.
81        """
82        return self._deactivate(debug=(debug or self._debug), **self._kwargs)

Deactivate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be deactivated.

target_path: pathlib.Path
85    @property
86    def target_path(self) -> pathlib.Path:
87        """
88        Return the target site-packages path for this virtual environment.
89        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
90        (e.g. Python 3.10 and Python 3.7).
91        """
92        from meerschaum.utils.venv import venv_target_path
93        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)

Return the target site-packages path for this virtual environment. A meerschaum.utils.venv.Venv may have one virtual environment per minor Python version (e.g. Python 3.10 and Python 3.7).

root_path: pathlib.Path
 96    @property
 97    def root_path(self) -> pathlib.Path:
 98        """
 99        Return the top-level path for this virtual environment.
100        """
101        from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
102        if self._venv is None:
103            return self.target_path.parent
104        return VIRTENV_RESOURCES_PATH / self._venv

Return the top-level path for this virtual environment.

def activate_venv( venv: Optional[str] = 'mrsm', color: bool = True, force: bool = False, debug: bool = False, **kw) -> bool:
 36def activate_venv(
 37    venv: Optional[str] = 'mrsm',
 38    color: bool = True,
 39    force: bool = False,
 40    debug: bool = False,
 41    **kw
 42) -> bool:
 43    """
 44    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 45
 46    Parameters
 47    ----------
 48    venv: Optional[str], default 'mrsm'
 49        The virtual environment to activate.
 50
 51    color: bool, default True
 52        If `True`, include color in debug text.
 53
 54    force: bool, default False
 55        If `True`, do not exit early even if the venv is currently active.
 56
 57    debug: bool, default False
 58        Verbosity toggle.
 59
 60    Returns
 61    -------
 62    A bool indicating whether the virtual environment was successfully activated.
 63
 64    """
 65    thread_id = get_ident()
 66    if active_venvs_order and active_venvs_order[0] == venv:
 67        if not force:
 68            return True
 69    import sys
 70    import os
 71    if venv is not None:
 72        init_venv(venv=venv, debug=debug)
 73    with LOCKS['active_venvs']:
 74        if thread_id not in threads_active_venvs:
 75            threads_active_venvs[thread_id] = {}
 76        active_venvs.add(venv)
 77        if venv not in threads_active_venvs[thread_id]:
 78            threads_active_venvs[thread_id][venv] = 1
 79        else:
 80            threads_active_venvs[thread_id][venv] += 1
 81
 82        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 83        if not target_path.exists():
 84            init_venv(venv=venv, force=True, debug=debug)
 85        if not target_path.exists():
 86            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 87        target = target_path.as_posix()
 88
 89        if venv in active_venvs_order:
 90            try:
 91                sys.path.remove(target)
 92            except Exception:
 93                pass
 94            try:
 95                active_venvs_order.remove(venv)
 96            except Exception:
 97                pass
 98        if venv is not None:
 99            sys.path.insert(0, target)
100        else:
101            if sys.path and sys.path[0] in (os.getcwd(), ''):
102                sys.path.insert(1, target)
103            else:
104                sys.path.insert(0, target)
105        try:
106            active_venvs_order.insert(0, venv)
107        except Exception:
108            pass
109
110    return True

Create a virtual environment (if it doesn't exist) and add it to sys.path if necessary.

Parameters
  • venv (Optional[str], default 'mrsm'): The virtual environment to activate.
  • color (bool, default True): If True, include color in debug text.
  • force (bool, default False): If True, do not exit early even if the venv is currently active.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment was successfully activated.
def deactivate_venv( venv: str = 'mrsm', color: bool = True, debug: bool = False, previously_active_venvs: Union[set[str], List[str], NoneType] = None, force: bool = False, **kw) -> bool:
113def deactivate_venv(
114    venv: str = 'mrsm',
115    color: bool = True,
116    debug: bool = False,
117    previously_active_venvs: Union['set[str]', List[str], None] = None,
118    force: bool = False,
119    **kw
120) -> bool:
121    """
122    Remove a virtual environment from `sys.path` (if it's been activated).
123
124    Parameters
125    ----------
126    venv: str, default 'mrsm'
127        The virtual environment to deactivate.
128
129    color: bool, default True
130        If `True`, include color in debug text.
131
132    debug: bool, default False
133        Verbosity toggle.
134
135    previously_active_venvs: Union[Set[str], List[str], None]
136        If provided, skip deactivating if a virtual environment is in this iterable.
137
138    force: bool, default False
139        If `True`, forcibly deactivate the virtual environment.
140        This may cause issues with other threads, so be careful!
141
142    Returns
143    -------
144    Return a bool indicating whether the virtual environment was successfully deactivated.
145
146    """
147    import sys
148    thread_id = get_ident()
149    if venv is None:
150        if venv in active_venvs:
151            active_venvs.remove(venv)
152        return True
153
154    if previously_active_venvs and venv in previously_active_venvs and not force:
155        return True
156
157    with LOCKS['active_venvs']:
158        if venv in threads_active_venvs.get(thread_id, {}):
159            new_count = threads_active_venvs[thread_id][venv] - 1
160            if new_count > 0 and not force:
161                threads_active_venvs[thread_id][venv] = new_count
162                return True
163            else:
164                del threads_active_venvs[thread_id][venv]
165
166        if not force:
167            for other_thread_id, other_venvs in threads_active_venvs.items():
168                if other_thread_id == thread_id:
169                    continue
170                if venv in other_venvs:
171                    return True
172        else:
173            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
174            for other_thread_id in to_delete:
175                del threads_active_venvs[other_thread_id]
176
177        if venv in active_venvs:
178            active_venvs.remove(venv)
179
180    if sys.path is None:
181        return False
182
183    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
184    with LOCKS['sys.path']:
185        if target in sys.path:
186            try:
187                sys.path.remove(target)
188            except Exception:
189                pass
190            try:
191                active_venvs_order.remove(venv)
192            except Exception:
193                pass
194
195    return True

Remove a virtual environment from sys.path (if it's been activated).

Parameters
  • venv (str, default 'mrsm'): The virtual environment to deactivate.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
  • previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
  • force (bool, default False): If True, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
  • Return a bool indicating whether the virtual environment was successfully deactivated.
def get_module_venv(module) -> Optional[str]:
830def get_module_venv(module) -> Union[str, None]:
831    """
832    Return the virtual environment where an imported module is installed.
833
834    Parameters
835    ----------
836    module: ModuleType
837        The imported module to inspect.
838
839    Returns
840    -------
841    The name of a venv or `None`.
842    """
843    import pathlib
844    from meerschaum.config.paths import VIRTENV_RESOURCES_PATH
845    module_path = pathlib.Path(module.__file__).resolve()
846    try:
847        rel_path = module_path.relative_to(VIRTENV_RESOURCES_PATH)
848    except ValueError:
849        return None
850
851    return rel_path.as_posix().split('/', maxsplit=1)[0]

Return the virtual environment where an imported module is installed.

Parameters
  • module (ModuleType): The imported module to inspect.
Returns
  • The name of a venv or None.
def get_venvs() -> List[str]:
813def get_venvs() -> List[str]:
814    """
815    Return a list of all the virtual environments.
816    """
817    import os
818    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
819    venvs = []
820    for filename in os.listdir(VIRTENV_RESOURCES_PATH):
821        path = VIRTENV_RESOURCES_PATH / filename
822        if not path.is_dir():
823            continue
824        if not venv_exists(filename):
825            continue
826        venvs.append(filename)
827    return venvs

Return a list of all the virtual environments.

def init_venv( venv: str = 'mrsm', verify: bool = True, force: bool = False, debug: bool = False) -> bool:
349def init_venv(
350    venv: str = 'mrsm',
351    verify: bool = True,
352    force: bool = False,
353    debug: bool = False,
354) -> bool:
355    """
356    Initialize the virtual environment.
357
358    Parameters
359    ----------
360    venv: str, default 'mrsm'
361        The name of the virtual environment to create.
362
363    verify: bool, default True
364        If `True`, verify that the virtual environment is in the expected state.
365
366    force: bool, default False
367        If `True`, recreate the virtual environment, even if already initalized.
368
369    Returns
370    -------
371    A `bool` indicating success.
372    """
373    if not force and venv in verified_venvs:
374        return True
375    if not force and venv_exists(venv, debug=debug):
376        if verify:
377            verify_venv(venv, debug=debug)
378            verified_venvs.add(venv)
379        return True
380
381    import io
382    from contextlib import redirect_stdout
383    import sys
384    import platform
385    import os
386    import pathlib
387    import shutil
388    import time
389
390    from meerschaum.config.static import STATIC_CONFIG
391    from meerschaum.config._paths import (
392        VIRTENV_RESOURCES_PATH,
393        VENVS_CACHE_RESOURCES_PATH,
394    )
395    from meerschaum.utils.packages import is_uv_enabled
396
397    venv_path = VIRTENV_RESOURCES_PATH / venv
398    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
399    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
400    lock_path = VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
401    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
402
403    def update_lock(active: bool):
404        try:
405            if not active:
406                if debug:
407                    print(f"Releasing lock: '{lock_path}'")
408                lock_path.unlink()
409            else:
410                if debug:
411                    print(f"Acquiring lock: '{lock_path}'")
412                lock_path.touch()
413        except Exception:
414            pass
415
416    def wait_for_lock():
417        if platform.system() == 'Windows':
418            return
419        max_lock_seconds = 30.0
420        sleep_message_seconds = 5.0
421        step_sleep_seconds = 0.1
422        init_venv_check_start = time.perf_counter()
423        last_print = init_venv_check_start
424        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
425            if not lock_path.exists():
426                break
427
428            now = time.perf_counter()
429            if debug or (now - last_print) > sleep_message_seconds:
430                print(f"Lock exists for venv '{venv}', sleeping...")
431                last_print = now
432            time.sleep(step_sleep_seconds)
433        update_lock(False)
434
435    if (
436        not force
437        and venv == 'mrsm'
438        and os.environ.get(work_dir_env_var, None) is not None
439        and docker_home_venv_path.exists()
440    ):
441        wait_for_lock()
442        shutil.move(docker_home_venv_path, venv_path)
443        if verify:
444            verify_venv(venv, debug=debug)
445            verified_venvs.add(venv)
446        return True
447
448    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
449    global tried_virtualenv
450    try:
451        import venv as _venv
452        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
453        virtualenv = None
454    except ImportError:
455        _venv = None
456        uv = None
457        virtualenv = None
458
459    _venv_success = False
460    temp_vtp = VENVS_CACHE_RESOURCES_PATH / str(venv)
461    ### NOTE: Disable site-packages movement for now.
462    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
463
464    wait_for_lock()
465    update_lock(True)
466
467    if rename_vtp:
468        if debug:
469            print(f"Moving '{vtp}' to '{temp_vtp}'...")
470        shutil.move(vtp, temp_vtp)
471
472    if uv is not None:
473        _venv_success = run_python_package(
474            'uv',
475            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
476            venv=None,
477            env=_get_pip_os_env(),
478            debug=debug,
479        ) == 0
480
481    if _venv is not None and not _venv_success:
482        f = io.StringIO()
483        with redirect_stdout(f):
484            _venv_success = run_python_package(
485                'venv',
486                [venv_path.as_posix()] + (
487                    ['--symlinks']
488                    if platform.system() != 'Windows'
489                    else []
490                ),
491                venv=None, debug=debug
492            ) == 0
493        if not _venv_success:
494            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
495        if not venv_exists(venv, debug=debug):
496            _venv = None
497    if not _venv_success:
498        virtualenv = attempt_import(
499            'virtualenv',
500            venv=None,
501            lazy=False,
502            install=(not tried_virtualenv),
503            warn=False,
504            check_update=False,
505            color=False,
506            debug=debug,
507        )
508        if virtualenv is None:
509            print(
510                "Failed to import `venv` or `virtualenv`! "
511                + "Please install `virtualenv` via pip then restart Meerschaum."
512            )
513            if rename_vtp and temp_vtp.exists():
514                if debug:
515                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
516                shutil.move(temp_vtp, vtp)
517            update_lock(False)
518            return False
519
520        tried_virtualenv = True
521        try:
522            python_folder = (
523                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
524            )
525            dist_packages_path = (
526                VIRTENV_RESOURCES_PATH /
527                venv / 'local' / 'lib' / python_folder / 'dist-packages'
528            )
529            local_bin_path = VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
530            bin_path = VIRTENV_RESOURCES_PATH / venv / 'bin'
531            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
532            if bin_path.exists():
533                try:
534                    shutil.rmtree(bin_path)
535                except Exception:
536                    import traceback
537                    traceback.print_exc()
538            virtualenv.cli_run([venv_path.as_posix()])
539            if dist_packages_path.exists():
540                vtp.mkdir(exist_ok=True, parents=True)
541                for file_path in dist_packages_path.glob('*'):
542                    shutil.move(file_path, vtp)
543                shutil.rmtree(dist_packages_path)
544                #  shutil.move(dist_packages_path, vtp)
545                bin_path.mkdir(exist_ok=True, parents=True)
546                for file_path in local_bin_path.glob('*'):
547                    shutil.move(file_path, bin_path)
548                #  shutil.move(local_bin_path, bin_path)
549                shutil.rmtree(local_bin_path)
550
551        except Exception:
552            import traceback
553            traceback.print_exc()
554            if rename_vtp and temp_vtp.exists():
555                shutil.move(temp_vtp, vtp)
556            update_lock(False)
557            return False
558    if verify:
559        verify_venv(venv, debug=debug)
560        verified_venvs.add(venv)
561
562    if rename_vtp and temp_vtp.exists():
563        if debug:
564            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
565        shutil.move(temp_vtp, vtp)
566
567    update_lock(False)
568    return True

Initialize the virtual environment.

Parameters
  • venv (str, default 'mrsm'): The name of the virtual environment to create.
  • verify (bool, default True): If True, verify that the virtual environment is in the expected state.
  • force (bool, default False): If True, recreate the virtual environment, even if already initalized.
Returns
  • A bool indicating success.
def inside_venv() -> bool:
800def inside_venv() -> bool:
801    """
802    Determine whether current Python interpreter is running inside a virtual environment.
803    """
804    import sys
805    return (
806        hasattr(sys, 'real_prefix') or (
807            hasattr(sys, 'base_prefix')
808                and sys.base_prefix != sys.prefix
809        )
810    )

Determine whether current Python interpreter is running inside a virtual environment.

def is_venv_active(venv: str = 'mrsm', color: bool = True, debug: bool = False) -> bool:
198def is_venv_active(
199    venv: str = 'mrsm',
200    color : bool = True,
201    debug: bool = False
202) -> bool:
203    """
204    Check if a virtual environment is active.
205
206    Parameters
207    ----------
208    venv: str, default 'mrsm'
209        The virtual environment to check.
210
211    color: bool, default True
212        If `True`, include color in debug text.
213
214    debug: bool, default False
215        Verbosity toggle.
216
217    Returns
218    -------
219    A bool indicating whether the virtual environment `venv` is active.
220
221    """
222    return venv in active_venvs

Check if a virtual environment is active.

Parameters
  • venv (str, default 'mrsm'): The virtual environment to check.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment venv is active.
def venv_exec( code: str, venv: Optional[str] = 'mrsm', env: Optional[Dict[str, str]] = None, with_extras: bool = False, as_proc: bool = False, capture_output: bool = True, debug: bool = False) -> Union[bool, Tuple[int, bytes, bytes], subprocess.Popen]:
596def venv_exec(
597    code: str,
598    venv: Optional[str] = 'mrsm',
599    env: Optional[Dict[str, str]] = None,
600    with_extras: bool = False,
601    as_proc: bool = False,
602    capture_output: bool = True,
603    debug: bool = False,
604) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
605    """
606    Execute Python code in a subprocess via a virtual environment's interpeter.
607    Return `True` if the code successfully executes, `False` on failure.
608
609    Parameters
610    ----------
611    code: str
612        The Python code to excecute.
613
614    venv: str, default 'mrsm'
615        The virtual environment to use to get the path for the Python executable.
616        If `venv` is `None`, use the default `sys.executable` path.
617
618    env: Optional[Dict[str, str]], default None
619        Optionally specify the environment variables for the subprocess.
620        Defaults to `os.environ`.
621
622    with_extras: bool, default False
623        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
624
625    as_proc: bool, default False
626        If `True`, return the `subprocess.Popen` object instead of executing.
627
628    Returns
629    -------
630    By default, return a bool indicating success.
631    If `as_proc` is `True`, return a `subprocess.Popen` object.
632    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
633
634    """
635    import os
636    import subprocess
637    import platform
638    from meerschaum.utils.debug import dprint
639    from meerschaum.utils.process import _child_processes
640
641    executable = venv_executable(venv=venv)
642    cmd_list = [executable, '-c', code]
643    if env is None:
644        env = os.environ
645    if debug:
646        dprint(str(cmd_list))
647    if not with_extras and not as_proc:
648        return subprocess.call(cmd_list, env=env) == 0
649
650    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
651    group_kwargs = (
652        {
653            'preexec_fn': os.setsid,
654        } if platform.system() != 'Windows'
655        else {
656            'creationflags': CREATE_NEW_PROCESS_GROUP,
657        }
658    )
659    process = subprocess.Popen(
660        cmd_list,
661        stdout=stdout,
662        stderr=stderr,
663        env=env,
664        **group_kwargs
665    )
666    if as_proc:
667        _child_processes.append(process)
668        return process
669    stdout, stderr = process.communicate()
670    exit_code = process.returncode
671    return exit_code, stdout, stderr

Execute Python code in a subprocess via a virtual environment's interpeter. Return True if the code successfully executes, False on failure.

Parameters
  • code (str): The Python code to excecute.
  • venv (str, default 'mrsm'): The virtual environment to use to get the path for the Python executable. If venv is None, use the default sys.executable path.
  • env (Optional[Dict[str, str]], default None): Optionally specify the environment variables for the subprocess. Defaults to os.environ.
  • with_extras (bool, default False): If True, return a tuple of the exit code, stdout bytes, and stderr bytes.
  • as_proc (bool, default False): If True, return the subprocess.Popen object instead of executing.
Returns
  • By default, return a bool indicating success.
  • If as_proc is True, return a subprocess.Popen object.
  • If with_extras is True, return a tuple of the exit code, stdout bytes, and stderr bytes.
def venv_executable(venv: Optional[str] = 'mrsm') -> str:
571def venv_executable(venv: Optional[str] = 'mrsm') -> str:
572    """
573    The Python interpreter executable for a given virtual environment.
574    """
575    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
576    import sys
577    import platform
578    return (
579        sys.executable if venv is None
580        else str(
581            VIRTENV_RESOURCES_PATH
582            / venv
583            / (
584                'bin' if platform.system() != 'Windows'
585                else 'Scripts'
586            ) / (
587                'python'
588                + str(sys.version_info.major)
589                + '.'
590                + str(sys.version_info.minor)
591            )
592        )
593    )

The Python interpreter executable for a given virtual environment.

def venv_exists(venv: Optional[str], debug: bool = False) -> bool:
674def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
675    """
676    Determine whether a virtual environment has been created.
677    """
678    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
679    return target_path.exists()

Determine whether a virtual environment has been created.

def venv_target_path( venv: Optional[str], allow_nonexistent: bool = False, debug: bool = False) -> pathlib.Path:
682def venv_target_path(
683    venv: Union[str, None],
684    allow_nonexistent: bool = False,
685    debug: bool = False,
686) -> 'pathlib.Path':
687    """
688    Return a virtual environment's site-package path.
689
690    Parameters
691    ----------
692    venv: Union[str, None]
693        The virtual environment for which a path should be returned.
694
695    allow_nonexistent: bool, default False
696        If `True`, return a path even if it does not exist.
697
698    Returns
699    -------
700    The `pathlib.Path` object for the virtual environment's path.
701
702    """
703    import os
704    import sys
705    import platform
706    import pathlib
707    import site
708    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
709    from meerschaum.config.static import STATIC_CONFIG
710
711    ### Check sys.path for a user-writable site-packages directory.
712    if venv is None:
713
714        ### Return the known value for the portable environment.
715        environment_runtime = STATIC_CONFIG['environment']['runtime']
716        if os.environ.get(environment_runtime, None) == 'portable':
717            python_version_folder = (
718                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
719            )
720            executable_path = pathlib.Path(sys.executable)
721            site_packages_path = (
722                (
723                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
724                ) if platform.system() != 'Windows' else (
725                    executable_path.parent / 'Lib' / 'site-packages'
726                )
727            )
728            if not site_packages_path.exists():
729                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
730            return site_packages_path
731
732        if not inside_venv():
733            user_site_packages = site.getusersitepackages()
734            if user_site_packages is None:
735                raise EnvironmentError("Could not determine user site packages.")
736
737            site_path = pathlib.Path(user_site_packages)
738            if not site_path.exists():
739
740                ### Windows does not have `os.geteuid()`.
741                if platform.system() == 'Windows' or os.geteuid() != 0:
742                    site_path.mkdir(parents=True, exist_ok=True)
743                    return site_path
744
745                ### Allow for dist-level paths (running as root).
746                for possible_dist in site.getsitepackages():
747                    dist_path = pathlib.Path(possible_dist)
748                    if not dist_path.exists():
749                        continue
750                    return dist_path
751                
752                raise EnvironmentError("Could not determine the dist-packages directory.")
753
754            return site_path
755
756    venv_root_path = (
757        (VIRTENV_RESOURCES_PATH / venv)
758        if venv is not None else pathlib.Path(sys.prefix)
759    )
760    target_path = venv_root_path
761
762    ### Ensure 'lib' or 'Lib' exists.
763    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
764    if not allow_nonexistent:
765        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
766            print(f"Failed to find lib directory for virtual environment '{venv}'.")
767            import traceback
768            traceback.print_stack()
769            sys.exit(1)
770    target_path = target_path / lib
771
772    ### Check if a 'python3.x' folder exists.
773    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
774    if target_path.exists():
775        target_path = (
776            (target_path / python_folder) if python_folder in os.listdir(target_path)
777            else target_path
778        )
779    else:
780        target_path = (
781            (target_path / python_folder) if platform.system() != 'Windows'
782            else target_path
783        )
784
785    ### Ensure 'site-packages' exists.
786    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
787        target_path = target_path / 'site-packages'
788    else:
789        import traceback
790        traceback.print_stack()
791        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
792        print("This may be because you are using a different Python version.")
793        print("Try deleting the following directory and restarting Meerschaum:")
794        print(VIRTENV_RESOURCES_PATH)
795        sys.exit(1)
796
797    return target_path

Return a virtual environment's site-package path.

Parameters
  • venv (Union[str, None]): The virtual environment for which a path should be returned.
  • allow_nonexistent (bool, default False): If True, return a path even if it does not exist.
Returns
  • The pathlib.Path object for the virtual environment's path.
def verify_venv(venv: str, debug: bool = False) -> None:
226def verify_venv(
227    venv: str,
228    debug: bool = False,
229) -> None:
230    """
231    Verify that the virtual environment matches the expected state.
232    """
233    import pathlib
234    import platform
235    import os
236    import shutil
237    import sys
238    from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
239    from meerschaum.utils.process import run_process
240    from meerschaum.utils.misc import make_symlink, is_symlink
241    from meerschaum.utils.warnings import warn
242
243    venv_path = VIRTENV_RESOURCES_PATH / venv
244    bin_path = venv_path / (
245        'bin' if platform.system() != 'Windows' else "Scripts"
246    )
247    current_python_versioned_name = (
248        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
249        + ('' if platform.system() != 'Windows' else '.exe')
250    )
251
252    if not (bin_path / current_python_versioned_name).exists():
253        init_venv(venv, verify=False, force=True, debug=debug)
254        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
255        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
256        if not current_python_in_venv_path.exists():
257            if is_symlink(current_python_in_venv_path):
258                try:
259                    current_python_in_venv_path.unlink()
260                except Exception as e:
261                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
262            try:
263                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
264            except Exception:
265                print(
266                    f"Unable to create symlink {current_python_in_venv_path} "
267                    + f"to {current_python_in_sys_path}."
268                )
269        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
270    else:
271        files_to_inspect = [current_python_versioned_name]
272
273    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
274        """
275        Return the version for the python binary at the given path.
276        """
277        try:
278            ### It might be a broken symlink, so skip on errors.
279            if debug:
280                print(f"Getting python version for {python_path}")
281            proc = run_process(
282                [str(python_path), '-V'],
283                as_proc=True,
284                capture_output=True,
285            )
286            stdout, stderr = proc.communicate(timeout=1.0)
287        except Exception as e:
288            ### E.g. the symlink may be broken.
289            if is_symlink(python_path):
290                try:
291                    python_path.unlink()
292                except Exception as _e:
293                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
294            return None
295        return stdout.decode('utf-8').strip().replace('Python ', '')
296
297    ### Ensure the versions are symlinked correctly.
298    for filename in files_to_inspect:
299        if not filename.startswith('python'):
300            continue
301        python_path = bin_path / filename
302        version = get_python_version(python_path)
303        if version is None:
304            continue
305        try:
306            major_version = version.split('.', maxsplit=1)[0]
307            minor_version = version.split('.', maxsplit=2)[1]
308        except IndexError:
309            return
310        python_versioned_name = (
311            'python' + major_version + '.' + minor_version
312            + ('' if platform.system() != 'Windows' else '.exe')
313        )
314
315        ### E.g. python3.10 actually links to Python 3.10.
316        if filename == python_versioned_name:
317            real_path = pathlib.Path(os.path.realpath(python_path))
318            if not real_path.exists():
319                try:
320                    python_path.unlink()
321                except Exception:
322                    pass
323                init_venv(venv, verify=False, force=True, debug=debug)
324                if not python_path.exists():
325                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
326
327            if python_path == real_path:
328                continue
329
330            try:
331                python_path.unlink()
332            except Exception:
333                pass
334            success, msg = make_symlink(real_path, python_path)
335            if not success:
336                warn(msg, color=False)
337            continue
338
339        python_versioned_path = bin_path / python_versioned_name
340        if python_versioned_path.exists():
341            ### Avoid circular symlinks.
342            if get_python_version(python_versioned_path) == version:
343                continue
344            python_versioned_path.unlink()
345        shutil.move(python_path, python_versioned_path)

Verify that the virtual environment matches the expected state.