meerschaum.utils.venv

Manage virtual environments.

  1#! /usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3# vim:fenc=utf-8
  4
  5"""
  6Manage virtual environments.
  7"""
  8
  9from __future__ import annotations
 10
 11import sys
 12import pathlib
 13
 14from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple
 15from meerschaum.utils.threading import RLock, get_ident
 16
 17__all__ = sorted([
 18    'activate_venv', 'deactivate_venv', 'init_venv',
 19    'inside_venv', 'is_venv_active', 'venv_exec',
 20    'venv_executable', 'venv_exists', 'venv_target_path',
 21    'Venv', 'get_venvs', 'verify_venv', 'get_module_venv',
 22])
 23__pdoc__ = {'Venv': True}
 24
 25LOCKS = {
 26    'sys.path': RLock(),
 27    'active_venvs': RLock(),
 28    'venvs_active_counts': RLock(),
 29}
 30
 31active_venvs = set()
 32active_venvs_counts: Dict[str, int] = {}
 33active_venvs_order: List[Optional[str]] = []
 34threads_active_venvs: Dict[int, 'set[str]'] = {}
 35CREATE_NEW_PROCESS_GROUP = 0x00000200
 36
 37
 38def activate_venv(
 39    venv: Optional[str] = 'mrsm',
 40    color: bool = True,
 41    force: bool = False,
 42    init_if_not_exists: bool = True,
 43    debug: bool = False,
 44    **kw
 45) -> bool:
 46    """
 47    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 48
 49    Parameters
 50    ----------
 51    venv: Optional[str], default 'mrsm'
 52        The virtual environment to activate.
 53
 54    color: bool, default True
 55        If `True`, include color in debug text.
 56
 57    init_if_not_exists: bool, default True
 58        If `True`, create the virtual environment if it does not exist.
 59
 60    force: bool, default False
 61        If `True`, do not exit early even if the venv is currently active.
 62
 63    debug: bool, default False
 64        Verbosity toggle.
 65
 66    Returns
 67    -------
 68    A bool indicating whether the virtual environment was successfully activated.
 69
 70    """
 71    thread_id = get_ident()
 72    import sys
 73    import os
 74    if venv is not None and init_if_not_exists:
 75        init_venv(venv=venv, debug=debug)
 76
 77    with LOCKS['active_venvs']:
 78        if thread_id not in threads_active_venvs:
 79            threads_active_venvs[thread_id] = {}
 80        active_venvs.add(venv)
 81        if venv not in threads_active_venvs[thread_id]:
 82            threads_active_venvs[thread_id][venv] = 1
 83        else:
 84            threads_active_venvs[thread_id][venv] += 1
 85
 86        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 87        if not target_path.exists() and venv is not None and init_if_not_exists:
 88            init_venv(venv=venv, force=True, debug=debug)
 89
 90        if not target_path.exists():
 91            if init_if_not_exists:
 92                return False
 93            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 94
 95        target = target_path.as_posix()
 96        if (
 97            active_venvs_order
 98            and active_venvs_order[0] == venv
 99            and target in sys.path
100            and not force
101        ):
102            return True
103
104        if venv in active_venvs_order:
105            try:
106                sys.path.remove(target)
107            except Exception:
108                pass
109            try:
110                active_venvs_order.remove(venv)
111            except Exception:
112                pass
113
114        if venv is not None:
115            sys.path.insert(0, target)
116        else:
117            if sys.path and sys.path[0] in (os.getcwd(), ''):
118                sys.path.insert(1, target)
119            else:
120                sys.path.insert(0, target)
121        try:
122            active_venvs_order.insert(0, venv)
123        except Exception:
124            pass
125
126    return True
127
128
129def deactivate_venv(
130    venv: str = 'mrsm',
131    color: bool = True,
132    debug: bool = False,
133    previously_active_venvs: Union['set[str]', List[str], None] = None,
134    force: bool = False,
135    **kw
136) -> bool:
137    """
138    Remove a virtual environment from `sys.path` (if it's been activated).
139
140    Parameters
141    ----------
142    venv: str, default 'mrsm'
143        The virtual environment to deactivate.
144
145    color: bool, default True
146        If `True`, include color in debug text.
147
148    debug: bool, default False
149        Verbosity toggle.
150
151    previously_active_venvs: Union[Set[str], List[str], None]
152        If provided, skip deactivating if a virtual environment is in this iterable.
153
154    force: bool, default False
155        If `True`, forcibly deactivate the virtual environment.
156        This may cause issues with other threads, so be careful!
157
158    Returns
159    -------
160    Return a bool indicating whether the virtual environment was successfully deactivated.
161
162    """
163    import sys
164    thread_id = get_ident()
165    if venv is None:
166        if venv in active_venvs:
167            active_venvs.remove(venv)
168        return True
169
170    if previously_active_venvs and venv in previously_active_venvs and not force:
171        return True
172
173    with LOCKS['active_venvs']:
174        if venv in threads_active_venvs.get(thread_id, {}):
175            new_count = threads_active_venvs[thread_id][venv] - 1
176            if new_count > 0 and not force:
177                threads_active_venvs[thread_id][venv] = new_count
178                return True
179            else:
180                del threads_active_venvs[thread_id][venv]
181
182        if not force:
183            for other_thread_id, other_venvs in threads_active_venvs.items():
184                if other_thread_id == thread_id:
185                    continue
186                if venv in other_venvs:
187                    return True
188        else:
189            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
190            for other_thread_id in to_delete:
191                del threads_active_venvs[other_thread_id]
192
193        if venv in active_venvs:
194            active_venvs.remove(venv)
195
196    if sys.path is None:
197        return False
198
199    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
200    with LOCKS['sys.path']:
201        if target in sys.path:
202            try:
203                sys.path.remove(target)
204            except Exception:
205                pass
206            try:
207                active_venvs_order.remove(venv)
208            except Exception:
209                pass
210
211    return True
212
213
214def is_venv_active(
215    venv: str = 'mrsm',
216    color : bool = True,
217    debug: bool = False
218) -> bool:
219    """
220    Check if a virtual environment is active.
221
222    Parameters
223    ----------
224    venv: str, default 'mrsm'
225        The virtual environment to check.
226
227    color: bool, default True
228        If `True`, include color in debug text.
229
230    debug: bool, default False
231        Verbosity toggle.
232
233    Returns
234    -------
235    A bool indicating whether the virtual environment `venv` is active.
236
237    """
238    return venv in active_venvs
239
240
241verified_venvs = set()
242def verify_venv(
243    venv: str,
244    debug: bool = False,
245) -> None:
246    """
247    Verify that the virtual environment matches the expected state.
248    """
249    import platform
250    import os
251    import shutil
252    import sys
253
254    import meerschaum.config.paths as paths
255    from meerschaum.utils.process import run_process
256    from meerschaum.utils.misc import make_symlink, is_symlink
257    from meerschaum.utils.warnings import warn
258
259    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
260    bin_path = venv_path / (
261        'bin' if platform.system() != 'Windows' else "Scripts"
262    )
263    current_python_versioned_name = (
264        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
265        + ('' if platform.system() != 'Windows' else '.exe')
266    )
267
268    if not (bin_path / current_python_versioned_name).exists():
269        init_venv(venv, verify=False, force=True, debug=debug)
270        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
271        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
272        if not current_python_in_venv_path.exists():
273            if is_symlink(current_python_in_venv_path):
274                try:
275                    current_python_in_venv_path.unlink()
276                except Exception as e:
277                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
278            try:
279                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
280            except Exception:
281                print(
282                    f"Unable to create symlink {current_python_in_venv_path} "
283                    + f"to {current_python_in_sys_path}."
284                )
285        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
286    else:
287        files_to_inspect = [current_python_versioned_name]
288
289    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
290        """
291        Return the version for the python binary at the given path.
292        """
293        try:
294            ### It might be a broken symlink, so skip on errors.
295            if debug:
296                print(f"Getting python version for {python_path}")
297            proc = run_process(
298                [str(python_path), '-V'],
299                as_proc=True,
300                capture_output=True,
301            )
302            stdout, stderr = proc.communicate(timeout=1.0)
303        except Exception as e:
304            ### E.g. the symlink may be broken.
305            if is_symlink(python_path):
306                try:
307                    python_path.unlink()
308                except Exception as _e:
309                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
310            return None
311        return stdout.decode('utf-8').strip().replace('Python ', '')
312
313    ### Ensure the versions are symlinked correctly.
314    for filename in files_to_inspect:
315        if not filename.startswith('python'):
316            continue
317        python_path = bin_path / filename
318        version = get_python_version(python_path)
319        if version is None:
320            continue
321        try:
322            major_version = version.split('.', maxsplit=1)[0]
323            minor_version = version.split('.', maxsplit=2)[1]
324        except IndexError:
325            return
326        python_versioned_name = (
327            'python' + major_version + '.' + minor_version
328            + ('' if platform.system() != 'Windows' else '.exe')
329        )
330
331        ### E.g. python3.10 actually links to Python 3.10.
332        if filename == python_versioned_name:
333            try:
334                real_path = pathlib.Path(os.path.realpath(python_path))
335                real_path_exists = real_path.exists()
336            except Exception:
337                real_path_exists = False
338
339            if not real_path_exists:
340                try:
341                    python_path.unlink()
342                except Exception:
343                    pass
344                init_venv(venv, verify=False, force=True, debug=debug)
345                if not python_path.exists():
346                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
347
348            if python_path == real_path:
349                continue
350
351            try:
352                python_path.unlink()
353            except Exception:
354                pass
355            success, msg = make_symlink(real_path, python_path)
356            if not success:
357                warn(msg, color=False)
358            continue
359
360        python_versioned_path = bin_path / python_versioned_name
361        if python_versioned_path.exists():
362            ### Avoid circular symlinks.
363            if get_python_version(python_versioned_path) == version:
364                continue
365            python_versioned_path.unlink()
366        shutil.move(python_path, python_versioned_path)
367
368
369tried_virtualenv = False
370def init_venv(
371    venv: str = 'mrsm',
372    verify: bool = True,
373    force: bool = False,
374    debug: bool = False,
375) -> bool:
376    """
377    Initialize the virtual environment.
378
379    Parameters
380    ----------
381    venv: str, default 'mrsm'
382        The name of the virtual environment to create.
383
384    verify: bool, default True
385        If `True`, verify that the virtual environment is in the expected state.
386
387    force: bool, default False
388        If `True`, recreate the virtual environment, even if already initalized.
389
390    Returns
391    -------
392    A `bool` indicating success.
393    """
394    if not force and venv in verified_venvs:
395        return True
396    if not force and venv_exists(venv, debug=debug):
397        if verify:
398            verify_venv(venv, debug=debug)
399            verified_venvs.add(venv)
400        return True
401
402    import io
403    from contextlib import redirect_stdout
404    import sys
405    import platform
406    import os
407    import shutil
408    import time
409
410    import meerschaum.config.paths as paths
411    from meerschaum._internal.static import STATIC_CONFIG
412    from meerschaum.utils.packages import is_uv_enabled
413
414    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
415    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
416    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
417    lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
418    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
419
420    def update_lock(active: bool):
421        try:
422            if not active:
423                if debug:
424                    print(f"Releasing lock: '{lock_path}'")
425                lock_path.unlink()
426            else:
427                if debug:
428                    print(f"Acquiring lock: '{lock_path}'")
429                lock_path.touch()
430        except Exception:
431            pass
432
433    def wait_for_lock():
434        if platform.system() == 'Windows':
435            return
436        max_lock_seconds = 30.0
437        sleep_message_seconds = 5.0
438        step_sleep_seconds = 0.1
439        init_venv_check_start = time.perf_counter()
440        last_print = init_venv_check_start
441        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
442            if not lock_path.exists():
443                break
444
445            now = time.perf_counter()
446            if debug or (now - last_print) > sleep_message_seconds:
447                print(f"Lock exists for venv '{venv}', sleeping...")
448                last_print = now
449            time.sleep(step_sleep_seconds)
450        update_lock(False)
451
452    if (
453        not force
454        and venv == 'mrsm'
455        and os.environ.get(work_dir_env_var, None) is not None
456        and docker_home_venv_path.exists()
457    ):
458        wait_for_lock()
459        shutil.move(docker_home_venv_path, venv_path)
460        if verify:
461            verify_venv(venv, debug=debug)
462            verified_venvs.add(venv)
463        return True
464
465    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
466    global tried_virtualenv
467    try:
468        import venv as _venv
469        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
470        virtualenv = None
471    except ImportError:
472        _venv = None
473        uv = None
474        virtualenv = None
475
476    _venv_success = False
477    temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv)
478    ### NOTE: Disable site-packages movement for now.
479    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
480
481    wait_for_lock()
482    update_lock(True)
483
484    if rename_vtp:
485        if debug:
486            print(f"Moving '{vtp}' to '{temp_vtp}'...")
487        shutil.move(vtp, temp_vtp)
488
489    if uv is not None:
490        _venv_success = run_python_package(
491            'uv',
492            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
493            venv=None,
494            env=_get_pip_os_env(),
495            debug=debug,
496        ) == 0
497
498    if _venv is not None and not _venv_success:
499        f = io.StringIO()
500        with redirect_stdout(f):
501            _venv_success = run_python_package(
502                'venv',
503                [venv_path.as_posix()] + (
504                    ['--symlinks']
505                    if platform.system() != 'Windows'
506                    else []
507                ),
508                venv=None, debug=debug
509            ) == 0
510        if not _venv_success:
511            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
512        if not venv_exists(venv, debug=debug):
513            _venv = None
514    if not _venv_success:
515        virtualenv = attempt_import(
516            'virtualenv',
517            venv=None,
518            lazy=False,
519            install=(not tried_virtualenv),
520            warn=False,
521            check_update=False,
522            color=False,
523            debug=debug,
524        )
525        if virtualenv is None:
526            print(
527                "Failed to import `venv` or `virtualenv`! "
528                + "Please install `virtualenv` via pip then restart Meerschaum."
529            )
530            if rename_vtp and temp_vtp.exists():
531                if debug:
532                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
533                shutil.move(temp_vtp, vtp)
534            update_lock(False)
535            return False
536
537        tried_virtualenv = True
538        try:
539            python_folder = (
540                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
541            )
542            dist_packages_path = (
543                paths.VIRTENV_RESOURCES_PATH /
544                venv / 'local' / 'lib' / python_folder / 'dist-packages'
545            )
546            local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
547            bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin'
548            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
549            if bin_path.exists():
550                try:
551                    shutil.rmtree(bin_path)
552                except Exception:
553                    import traceback
554                    traceback.print_exc()
555            virtualenv.cli_run([venv_path.as_posix()])
556            if dist_packages_path.exists():
557                vtp.mkdir(exist_ok=True, parents=True)
558                for file_path in dist_packages_path.glob('*'):
559                    shutil.move(file_path, vtp)
560                shutil.rmtree(dist_packages_path)
561                #  shutil.move(dist_packages_path, vtp)
562                bin_path.mkdir(exist_ok=True, parents=True)
563                for file_path in local_bin_path.glob('*'):
564                    shutil.move(file_path, bin_path)
565                #  shutil.move(local_bin_path, bin_path)
566                shutil.rmtree(local_bin_path)
567
568        except Exception:
569            import traceback
570            traceback.print_exc()
571            if rename_vtp and temp_vtp.exists():
572                shutil.move(temp_vtp, vtp)
573            update_lock(False)
574            return False
575    if verify:
576        verify_venv(venv, debug=debug)
577        verified_venvs.add(venv)
578
579    if rename_vtp and temp_vtp.exists():
580        if debug:
581            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
582        shutil.move(temp_vtp, vtp)
583
584    update_lock(False)
585    return True
586
587
588def venv_executable(venv: Optional[str] = 'mrsm') -> str:
589    """
590    The Python interpreter executable for a given virtual environment.
591    """
592    import sys
593    import platform
594    import meerschaum.config.paths as paths
595    return (
596        sys.executable if venv is None
597        else str(
598            paths.VIRTENV_RESOURCES_PATH
599            / venv
600            / (
601                'bin' if platform.system() != 'Windows'
602                else 'Scripts'
603            ) / (
604                'python'
605                + str(sys.version_info.major)
606                + '.'
607                + str(sys.version_info.minor)
608            )
609        )
610    )
611
612
613def venv_exec(
614    code: str,
615    venv: Optional[str] = 'mrsm',
616    env: Optional[Dict[str, str]] = None,
617    with_extras: bool = False,
618    as_proc: bool = False,
619    capture_output: bool = True,
620    debug: bool = False,
621) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
622    """
623    Execute Python code in a subprocess via a virtual environment's interpeter.
624    Return `True` if the code successfully executes, `False` on failure.
625
626    Parameters
627    ----------
628    code: str
629        The Python code to excecute.
630
631    venv: str, default 'mrsm'
632        The virtual environment to use to get the path for the Python executable.
633        If `venv` is `None`, use the default `sys.executable` path.
634
635    env: Optional[Dict[str, str]], default None
636        Optionally specify the environment variables for the subprocess.
637        Defaults to `os.environ`.
638
639    with_extras: bool, default False
640        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
641
642    as_proc: bool, default False
643        If `True`, return the `subprocess.Popen` object instead of executing.
644
645    Returns
646    -------
647    By default, return a bool indicating success.
648    If `as_proc` is `True`, return a `subprocess.Popen` object.
649    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
650
651    """
652    import os
653    import subprocess
654    import platform
655    from meerschaum.utils.debug import dprint
656    from meerschaum.utils.process import _child_processes
657
658    executable = venv_executable(venv=venv)
659    cmd_list = [executable, '-c', code]
660    if env is None:
661        env = os.environ
662    if debug:
663        dprint(str(cmd_list))
664    if not with_extras and not as_proc:
665        return subprocess.call(cmd_list, env=env) == 0
666
667    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
668    group_kwargs = (
669        {
670            'preexec_fn': os.setsid,
671        } if platform.system() != 'Windows'
672        else {
673            'creationflags': CREATE_NEW_PROCESS_GROUP,
674        }
675    )
676    process = subprocess.Popen(
677        cmd_list,
678        stdout=stdout,
679        stderr=stderr,
680        stdin=sys.stdin,
681        env=env,
682        **group_kwargs
683    )
684    if as_proc:
685        _child_processes.append(process)
686        return process
687    stdout, stderr = process.communicate()
688    exit_code = process.returncode
689    return exit_code, stdout, stderr
690
691
692def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
693    """
694    Determine whether a virtual environment has been created.
695    """
696    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
697    return target_path.exists()
698
699
700def venv_target_path(
701    venv: Union[str, None],
702    allow_nonexistent: bool = False,
703    debug: bool = False,
704) -> pathlib.Path:
705    """
706    Return a virtual environment's site-package path.
707
708    Parameters
709    ----------
710    venv: Union[str, None]
711        The virtual environment for which a path should be returned.
712
713    allow_nonexistent: bool, default False
714        If `True`, return a path even if it does not exist.
715
716    Returns
717    -------
718    The `pathlib.Path` object for the virtual environment's path.
719
720    """
721    import os
722    import sys
723    import platform
724    import site
725
726    import meerschaum.config.paths as paths
727    from meerschaum._internal.static import STATIC_CONFIG
728
729    ### Check sys.path for a user-writable site-packages directory.
730    if venv is None:
731
732        ### Return the known value for the portable environment.
733        environment_runtime = STATIC_CONFIG['environment']['runtime']
734        if os.environ.get(environment_runtime, None) == 'portable':
735            python_version_folder = (
736                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
737            )
738            executable_path = pathlib.Path(sys.executable)
739            site_packages_path = (
740                (
741                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
742                ) if platform.system() != 'Windows' else (
743                    executable_path.parent / 'Lib' / 'site-packages'
744                )
745            )
746            if not site_packages_path.exists():
747                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
748            return site_packages_path
749
750        if not inside_venv():
751            user_site_packages = site.getusersitepackages()
752            if user_site_packages is None:
753                raise EnvironmentError("Could not determine user site packages.")
754
755            site_path = pathlib.Path(user_site_packages)
756            if not site_path.exists():
757
758                ### Windows does not have `os.geteuid()`.
759                if platform.system() == 'Windows' or os.geteuid() != 0:
760                    site_path.mkdir(parents=True, exist_ok=True)
761                    return site_path
762
763                ### Allow for dist-level paths (running as root).
764                for possible_dist in site.getsitepackages():
765                    dist_path = pathlib.Path(possible_dist)
766                    if not dist_path.exists():
767                        continue
768                    return dist_path
769                
770                raise EnvironmentError("Could not determine the dist-packages directory.")
771
772            return site_path
773
774    venv_root_path = (
775        (paths.VIRTENV_RESOURCES_PATH / venv)
776        if venv is not None
777        else pathlib.Path(sys.prefix)
778    )
779    target_path = venv_root_path
780
781    ### Ensure 'lib' or 'Lib' exists.
782    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
783    if not allow_nonexistent:
784        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
785            print(f"Failed to find lib directory for virtual environment '{venv}'.")
786            import traceback
787            traceback.print_stack()
788            sys.exit(1)
789    target_path = target_path / lib
790
791    ### Check if a 'python3.x' folder exists.
792    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
793    if target_path.exists():
794        target_path = (
795            (target_path / python_folder)
796            if python_folder in os.listdir(target_path)
797            else target_path
798        )
799    else:
800        target_path = (
801            (target_path / python_folder)
802            if platform.system() != 'Windows'
803            else target_path
804        )
805
806    ### Ensure 'site-packages' exists.
807    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
808        target_path = target_path / 'site-packages'
809    else:
810        import traceback
811        traceback.print_stack()
812        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
813        print("This may be because you are using a different Python version.")
814        print("Try deleting the following directory and restarting Meerschaum:")
815        print(paths.VIRTENV_RESOURCES_PATH)
816        sys.exit(1)
817
818    return target_path
819
820
821def inside_venv() -> bool:
822    """
823    Determine whether current Python interpreter is running inside a virtual environment.
824    """
825    import sys
826    return (
827        hasattr(sys, 'real_prefix') or (
828            hasattr(sys, 'base_prefix')
829                and sys.base_prefix != sys.prefix
830        )
831    )
832
833
834def get_venvs() -> List[str]:
835    """
836    Return a list of all the virtual environments.
837    """
838    import os
839    import meerschaum.config.paths as paths
840    venvs = []
841    for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH):
842        path = paths.VIRTENV_RESOURCES_PATH / filename
843        if not path.is_dir():
844            continue
845        if not venv_exists(filename):
846            continue
847        venvs.append(filename)
848    return venvs
849
850
851def get_module_venv(module) -> Union[str, None]:
852    """
853    Return the virtual environment where an imported module is installed.
854
855    Parameters
856    ----------
857    module: ModuleType
858        The imported module to inspect.
859
860    Returns
861    -------
862    The name of a venv or `None`.
863    """
864    import meerschaum.config.paths as paths
865    module_path = pathlib.Path(module.__file__).resolve()
866    try:
867        rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH)
868    except ValueError:
869        return None
870
871    return rel_path.as_posix().split('/', maxsplit=1)[0]
872
873
874from meerschaum.utils.venv._Venv import Venv
class Venv:
 19class Venv:
 20    """
 21    Manage a virtual enviroment's activation status.
 22
 23    Examples
 24    --------
 25    >>> from meerschaum.plugins import Plugin
 26    >>> with Venv('mrsm') as venv:
 27    ...     import pandas
 28    >>> with Venv(Plugin('noaa')) as venv:
 29    ...     import requests
 30    >>> venv = Venv('mrsm')
 31    >>> venv.activate()
 32    True
 33    >>> venv.deactivate()
 34    True
 35    >>> 
 36    """
 37
 38    def __init__(
 39        self,
 40        venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm',
 41        init_if_not_exists: bool = True,
 42        debug: bool = False,
 43    ) -> None:
 44        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
 45        ### For some weird threading issue,
 46        ### we can't use `isinstance` here.
 47        if '_Plugin' in str(type(venv)):
 48            self._venv = venv.name
 49            self._activate = venv.activate_venv
 50            self._deactivate = venv.deactivate_venv
 51            self._kwargs = {}
 52        else:
 53            self._venv = venv
 54            self._activate = activate_venv
 55            self._deactivate = deactivate_venv
 56            self._kwargs = {'venv': venv}
 57        self._debug = debug
 58        self._init_if_not_exists = init_if_not_exists
 59        ### In case someone calls `deactivate()` before `activate()`.
 60        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 61
 62
 63    def activate(self, debug: bool = False) -> bool:
 64        """
 65        Activate this virtual environment.
 66        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 67        will also be activated.
 68        """
 69        from meerschaum.utils.venv import active_venvs, init_venv
 70        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
 71        try:
 72            return self._activate(
 73                debug=(debug or self._debug),
 74                init_if_not_exists=self._init_if_not_exists,
 75                **self._kwargs
 76            )
 77        except OSError as e:
 78            if self._init_if_not_exists:
 79                if not init_venv(self._venv, force=True):
 80                    raise e
 81        return self._activate(
 82            debug=(debug or self._debug),
 83            init_if_not_exists=self._init_if_not_exists,
 84            **self._kwargs
 85        )
 86
 87
 88    def deactivate(self, debug: bool = False) -> bool:
 89        """
 90        Deactivate this virtual environment.
 91        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
 92        will also be deactivated.
 93        """
 94        return self._deactivate(debug=(debug or self._debug), **self._kwargs)
 95
 96
 97    @property
 98    def target_path(self) -> pathlib.Path:
 99        """
100        Return the target site-packages path for this virtual environment.
101        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
102        (e.g. Python 3.10 and Python 3.7).
103        """
104        from meerschaum.utils.venv import venv_target_path
105        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)
106
107
108    @property
109    def root_path(self) -> pathlib.Path:
110        """
111        Return the top-level path for this virtual environment.
112        """
113        import meerschaum.config.paths as paths
114        if self._venv is None:
115            return self.target_path.parent
116        return paths.VIRTENV_RESOURCES_PATH / self._venv
117
118
119    def __enter__(self) -> None:
120        self.activate(debug=self._debug)
121
122
123    def __exit__(self, exc_type, exc_value, exc_traceback) -> None:
124        self.deactivate(debug=self._debug)
125
126
127    def __str__(self) -> str:
128        quote = "'" if self._venv is not None else ""
129        return "Venv(" + quote + str(self._venv) + quote + ")"
130
131
132    def __repr__(self) -> str:
133        return self.__str__()

Manage a virtual enviroment's activation status.

Examples
>>> from meerschaum.plugins import Plugin
>>> with Venv('mrsm') as venv:
...     import pandas
>>> with Venv(Plugin('noaa')) as venv:
...     import requests
>>> venv = Venv('mrsm')
>>> venv.activate()
True
>>> venv.deactivate()
True
>>>
Venv( venv: Union[str, meerschaum.Plugin, NoneType] = 'mrsm', init_if_not_exists: bool = True, debug: bool = False)
38    def __init__(
39        self,
40        venv: Union[str, 'mrsm.core.Plugin', None] = 'mrsm',
41        init_if_not_exists: bool = True,
42        debug: bool = False,
43    ) -> None:
44        from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
45        ### For some weird threading issue,
46        ### we can't use `isinstance` here.
47        if '_Plugin' in str(type(venv)):
48            self._venv = venv.name
49            self._activate = venv.activate_venv
50            self._deactivate = venv.deactivate_venv
51            self._kwargs = {}
52        else:
53            self._venv = venv
54            self._activate = activate_venv
55            self._deactivate = deactivate_venv
56            self._kwargs = {'venv': venv}
57        self._debug = debug
58        self._init_if_not_exists = init_if_not_exists
59        ### In case someone calls `deactivate()` before `activate()`.
60        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
def activate(self, debug: bool = False) -> bool:
63    def activate(self, debug: bool = False) -> bool:
64        """
65        Activate this virtual environment.
66        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
67        will also be activated.
68        """
69        from meerschaum.utils.venv import active_venvs, init_venv
70        self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
71        try:
72            return self._activate(
73                debug=(debug or self._debug),
74                init_if_not_exists=self._init_if_not_exists,
75                **self._kwargs
76            )
77        except OSError as e:
78            if self._init_if_not_exists:
79                if not init_venv(self._venv, force=True):
80                    raise e
81        return self._activate(
82            debug=(debug or self._debug),
83            init_if_not_exists=self._init_if_not_exists,
84            **self._kwargs
85        )

Activate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be activated.

def deactivate(self, debug: bool = False) -> bool:
88    def deactivate(self, debug: bool = False) -> bool:
89        """
90        Deactivate this virtual environment.
91        If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
92        will also be deactivated.
93        """
94        return self._deactivate(debug=(debug or self._debug), **self._kwargs)

Deactivate this virtual environment. If a meerschaum.plugins.Plugin was provided, its dependent virtual environments will also be deactivated.

target_path: pathlib.Path
 97    @property
 98    def target_path(self) -> pathlib.Path:
 99        """
100        Return the target site-packages path for this virtual environment.
101        A `meerschaum.utils.venv.Venv` may have one virtual environment per minor Python version
102        (e.g. Python 3.10 and Python 3.7).
103        """
104        from meerschaum.utils.venv import venv_target_path
105        return venv_target_path(venv=self._venv, allow_nonexistent=True, debug=self._debug)

Return the target site-packages path for this virtual environment. A meerschaum.utils.venv.Venv may have one virtual environment per minor Python version (e.g. Python 3.10 and Python 3.7).

root_path: pathlib.Path
108    @property
109    def root_path(self) -> pathlib.Path:
110        """
111        Return the top-level path for this virtual environment.
112        """
113        import meerschaum.config.paths as paths
114        if self._venv is None:
115            return self.target_path.parent
116        return paths.VIRTENV_RESOURCES_PATH / self._venv

Return the top-level path for this virtual environment.

def activate_venv( venv: Optional[str] = 'mrsm', color: bool = True, force: bool = False, init_if_not_exists: bool = True, debug: bool = False, **kw) -> bool:
 39def activate_venv(
 40    venv: Optional[str] = 'mrsm',
 41    color: bool = True,
 42    force: bool = False,
 43    init_if_not_exists: bool = True,
 44    debug: bool = False,
 45    **kw
 46) -> bool:
 47    """
 48    Create a virtual environment (if it doesn't exist) and add it to `sys.path` if necessary.
 49
 50    Parameters
 51    ----------
 52    venv: Optional[str], default 'mrsm'
 53        The virtual environment to activate.
 54
 55    color: bool, default True
 56        If `True`, include color in debug text.
 57
 58    init_if_not_exists: bool, default True
 59        If `True`, create the virtual environment if it does not exist.
 60
 61    force: bool, default False
 62        If `True`, do not exit early even if the venv is currently active.
 63
 64    debug: bool, default False
 65        Verbosity toggle.
 66
 67    Returns
 68    -------
 69    A bool indicating whether the virtual environment was successfully activated.
 70
 71    """
 72    thread_id = get_ident()
 73    import sys
 74    import os
 75    if venv is not None and init_if_not_exists:
 76        init_venv(venv=venv, debug=debug)
 77
 78    with LOCKS['active_venvs']:
 79        if thread_id not in threads_active_venvs:
 80            threads_active_venvs[thread_id] = {}
 81        active_venvs.add(venv)
 82        if venv not in threads_active_venvs[thread_id]:
 83            threads_active_venvs[thread_id][venv] = 1
 84        else:
 85            threads_active_venvs[thread_id][venv] += 1
 86
 87        target_path = venv_target_path(venv, debug=debug, allow_nonexistent=True)
 88        if not target_path.exists() and venv is not None and init_if_not_exists:
 89            init_venv(venv=venv, force=True, debug=debug)
 90
 91        if not target_path.exists():
 92            if init_if_not_exists:
 93                return False
 94            raise EnvironmentError(f"Could not activate virtual environment '{venv}'.")
 95
 96        target = target_path.as_posix()
 97        if (
 98            active_venvs_order
 99            and active_venvs_order[0] == venv
100            and target in sys.path
101            and not force
102        ):
103            return True
104
105        if venv in active_venvs_order:
106            try:
107                sys.path.remove(target)
108            except Exception:
109                pass
110            try:
111                active_venvs_order.remove(venv)
112            except Exception:
113                pass
114
115        if venv is not None:
116            sys.path.insert(0, target)
117        else:
118            if sys.path and sys.path[0] in (os.getcwd(), ''):
119                sys.path.insert(1, target)
120            else:
121                sys.path.insert(0, target)
122        try:
123            active_venvs_order.insert(0, venv)
124        except Exception:
125            pass
126
127    return True

Create a virtual environment (if it doesn't exist) and add it to sys.path if necessary.

Parameters
  • venv (Optional[str], default 'mrsm'): The virtual environment to activate.
  • color (bool, default True): If True, include color in debug text.
  • init_if_not_exists (bool, default True): If True, create the virtual environment if it does not exist.
  • force (bool, default False): If True, do not exit early even if the venv is currently active.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment was successfully activated.
def deactivate_venv( venv: str = 'mrsm', color: bool = True, debug: bool = False, previously_active_venvs: Union[set[str], List[str], NoneType] = None, force: bool = False, **kw) -> bool:
130def deactivate_venv(
131    venv: str = 'mrsm',
132    color: bool = True,
133    debug: bool = False,
134    previously_active_venvs: Union['set[str]', List[str], None] = None,
135    force: bool = False,
136    **kw
137) -> bool:
138    """
139    Remove a virtual environment from `sys.path` (if it's been activated).
140
141    Parameters
142    ----------
143    venv: str, default 'mrsm'
144        The virtual environment to deactivate.
145
146    color: bool, default True
147        If `True`, include color in debug text.
148
149    debug: bool, default False
150        Verbosity toggle.
151
152    previously_active_venvs: Union[Set[str], List[str], None]
153        If provided, skip deactivating if a virtual environment is in this iterable.
154
155    force: bool, default False
156        If `True`, forcibly deactivate the virtual environment.
157        This may cause issues with other threads, so be careful!
158
159    Returns
160    -------
161    Return a bool indicating whether the virtual environment was successfully deactivated.
162
163    """
164    import sys
165    thread_id = get_ident()
166    if venv is None:
167        if venv in active_venvs:
168            active_venvs.remove(venv)
169        return True
170
171    if previously_active_venvs and venv in previously_active_venvs and not force:
172        return True
173
174    with LOCKS['active_venvs']:
175        if venv in threads_active_venvs.get(thread_id, {}):
176            new_count = threads_active_venvs[thread_id][venv] - 1
177            if new_count > 0 and not force:
178                threads_active_venvs[thread_id][venv] = new_count
179                return True
180            else:
181                del threads_active_venvs[thread_id][venv]
182
183        if not force:
184            for other_thread_id, other_venvs in threads_active_venvs.items():
185                if other_thread_id == thread_id:
186                    continue
187                if venv in other_venvs:
188                    return True
189        else:
190            to_delete = [other_thread_id for other_thread_id in threads_active_venvs]
191            for other_thread_id in to_delete:
192                del threads_active_venvs[other_thread_id]
193
194        if venv in active_venvs:
195            active_venvs.remove(venv)
196
197    if sys.path is None:
198        return False
199
200    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
201    with LOCKS['sys.path']:
202        if target in sys.path:
203            try:
204                sys.path.remove(target)
205            except Exception:
206                pass
207            try:
208                active_venvs_order.remove(venv)
209            except Exception:
210                pass
211
212    return True

Remove a virtual environment from sys.path (if it's been activated).

Parameters
  • venv (str, default 'mrsm'): The virtual environment to deactivate.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
  • previously_active_venvs (Union[Set[str], List[str], None]): If provided, skip deactivating if a virtual environment is in this iterable.
  • force (bool, default False): If True, forcibly deactivate the virtual environment. This may cause issues with other threads, so be careful!
Returns
  • Return a bool indicating whether the virtual environment was successfully deactivated.
def get_module_venv(module) -> Optional[str]:
852def get_module_venv(module) -> Union[str, None]:
853    """
854    Return the virtual environment where an imported module is installed.
855
856    Parameters
857    ----------
858    module: ModuleType
859        The imported module to inspect.
860
861    Returns
862    -------
863    The name of a venv or `None`.
864    """
865    import meerschaum.config.paths as paths
866    module_path = pathlib.Path(module.__file__).resolve()
867    try:
868        rel_path = module_path.relative_to(paths.VIRTENV_RESOURCES_PATH)
869    except ValueError:
870        return None
871
872    return rel_path.as_posix().split('/', maxsplit=1)[0]

Return the virtual environment where an imported module is installed.

Parameters
  • module (ModuleType): The imported module to inspect.
Returns
  • The name of a venv or None.
def get_venvs() -> List[str]:
835def get_venvs() -> List[str]:
836    """
837    Return a list of all the virtual environments.
838    """
839    import os
840    import meerschaum.config.paths as paths
841    venvs = []
842    for filename in os.listdir(paths.VIRTENV_RESOURCES_PATH):
843        path = paths.VIRTENV_RESOURCES_PATH / filename
844        if not path.is_dir():
845            continue
846        if not venv_exists(filename):
847            continue
848        venvs.append(filename)
849    return venvs

Return a list of all the virtual environments.

def init_venv( venv: str = 'mrsm', verify: bool = True, force: bool = False, debug: bool = False) -> bool:
371def init_venv(
372    venv: str = 'mrsm',
373    verify: bool = True,
374    force: bool = False,
375    debug: bool = False,
376) -> bool:
377    """
378    Initialize the virtual environment.
379
380    Parameters
381    ----------
382    venv: str, default 'mrsm'
383        The name of the virtual environment to create.
384
385    verify: bool, default True
386        If `True`, verify that the virtual environment is in the expected state.
387
388    force: bool, default False
389        If `True`, recreate the virtual environment, even if already initalized.
390
391    Returns
392    -------
393    A `bool` indicating success.
394    """
395    if not force and venv in verified_venvs:
396        return True
397    if not force and venv_exists(venv, debug=debug):
398        if verify:
399            verify_venv(venv, debug=debug)
400            verified_venvs.add(venv)
401        return True
402
403    import io
404    from contextlib import redirect_stdout
405    import sys
406    import platform
407    import os
408    import shutil
409    import time
410
411    import meerschaum.config.paths as paths
412    from meerschaum._internal.static import STATIC_CONFIG
413    from meerschaum.utils.packages import is_uv_enabled
414
415    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
416    vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
417    docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
418    lock_path = paths.VENVS_CACHE_RESOURCES_PATH / (venv + '.lock')
419    work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
420
421    def update_lock(active: bool):
422        try:
423            if not active:
424                if debug:
425                    print(f"Releasing lock: '{lock_path}'")
426                lock_path.unlink()
427            else:
428                if debug:
429                    print(f"Acquiring lock: '{lock_path}'")
430                lock_path.touch()
431        except Exception:
432            pass
433
434    def wait_for_lock():
435        if platform.system() == 'Windows':
436            return
437        max_lock_seconds = 30.0
438        sleep_message_seconds = 5.0
439        step_sleep_seconds = 0.1
440        init_venv_check_start = time.perf_counter()
441        last_print = init_venv_check_start
442        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
443            if not lock_path.exists():
444                break
445
446            now = time.perf_counter()
447            if debug or (now - last_print) > sleep_message_seconds:
448                print(f"Lock exists for venv '{venv}', sleeping...")
449                last_print = now
450            time.sleep(step_sleep_seconds)
451        update_lock(False)
452
453    if (
454        not force
455        and venv == 'mrsm'
456        and os.environ.get(work_dir_env_var, None) is not None
457        and docker_home_venv_path.exists()
458    ):
459        wait_for_lock()
460        shutil.move(docker_home_venv_path, venv_path)
461        if verify:
462            verify_venv(venv, debug=debug)
463            verified_venvs.add(venv)
464        return True
465
466    from meerschaum.utils.packages import run_python_package, attempt_import, _get_pip_os_env
467    global tried_virtualenv
468    try:
469        import venv as _venv
470        uv = attempt_import('uv', venv=None, debug=debug) if is_uv_enabled() else None
471        virtualenv = None
472    except ImportError:
473        _venv = None
474        uv = None
475        virtualenv = None
476
477    _venv_success = False
478    temp_vtp = paths.VENVS_CACHE_RESOURCES_PATH / str(venv)
479    ### NOTE: Disable site-packages movement for now.
480    rename_vtp = False and vtp.exists() and not temp_vtp.exists()
481
482    wait_for_lock()
483    update_lock(True)
484
485    if rename_vtp:
486        if debug:
487            print(f"Moving '{vtp}' to '{temp_vtp}'...")
488        shutil.move(vtp, temp_vtp)
489
490    if uv is not None:
491        _venv_success = run_python_package(
492            'uv',
493            ['venv', venv_path.as_posix(), '-q', '--no-project', '--allow-existing', '--seed'],
494            venv=None,
495            env=_get_pip_os_env(),
496            debug=debug,
497        ) == 0
498
499    if _venv is not None and not _venv_success:
500        f = io.StringIO()
501        with redirect_stdout(f):
502            _venv_success = run_python_package(
503                'venv',
504                [venv_path.as_posix()] + (
505                    ['--symlinks']
506                    if platform.system() != 'Windows'
507                    else []
508                ),
509                venv=None, debug=debug
510            ) == 0
511        if not _venv_success:
512            print(f"Please install python3-venv.\n{f.getvalue()}\nFalling back to virtualenv...")
513        if not venv_exists(venv, debug=debug):
514            _venv = None
515    if not _venv_success:
516        virtualenv = attempt_import(
517            'virtualenv',
518            venv=None,
519            lazy=False,
520            install=(not tried_virtualenv),
521            warn=False,
522            check_update=False,
523            color=False,
524            debug=debug,
525        )
526        if virtualenv is None:
527            print(
528                "Failed to import `venv` or `virtualenv`! "
529                + "Please install `virtualenv` via pip then restart Meerschaum."
530            )
531            if rename_vtp and temp_vtp.exists():
532                if debug:
533                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
534                shutil.move(temp_vtp, vtp)
535            update_lock(False)
536            return False
537
538        tried_virtualenv = True
539        try:
540            python_folder = (
541                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
542            )
543            dist_packages_path = (
544                paths.VIRTENV_RESOURCES_PATH /
545                venv / 'local' / 'lib' / python_folder / 'dist-packages'
546            )
547            local_bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'local' / 'bin'
548            bin_path = paths.VIRTENV_RESOURCES_PATH / venv / 'bin'
549            vtp = venv_target_path(venv=venv, allow_nonexistent=True, debug=debug)
550            if bin_path.exists():
551                try:
552                    shutil.rmtree(bin_path)
553                except Exception:
554                    import traceback
555                    traceback.print_exc()
556            virtualenv.cli_run([venv_path.as_posix()])
557            if dist_packages_path.exists():
558                vtp.mkdir(exist_ok=True, parents=True)
559                for file_path in dist_packages_path.glob('*'):
560                    shutil.move(file_path, vtp)
561                shutil.rmtree(dist_packages_path)
562                #  shutil.move(dist_packages_path, vtp)
563                bin_path.mkdir(exist_ok=True, parents=True)
564                for file_path in local_bin_path.glob('*'):
565                    shutil.move(file_path, bin_path)
566                #  shutil.move(local_bin_path, bin_path)
567                shutil.rmtree(local_bin_path)
568
569        except Exception:
570            import traceback
571            traceback.print_exc()
572            if rename_vtp and temp_vtp.exists():
573                shutil.move(temp_vtp, vtp)
574            update_lock(False)
575            return False
576    if verify:
577        verify_venv(venv, debug=debug)
578        verified_venvs.add(venv)
579
580    if rename_vtp and temp_vtp.exists():
581        if debug:
582            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
583        shutil.move(temp_vtp, vtp)
584
585    update_lock(False)
586    return True

Initialize the virtual environment.

Parameters
  • venv (str, default 'mrsm'): The name of the virtual environment to create.
  • verify (bool, default True): If True, verify that the virtual environment is in the expected state.
  • force (bool, default False): If True, recreate the virtual environment, even if already initalized.
Returns
  • A bool indicating success.
def inside_venv() -> bool:
822def inside_venv() -> bool:
823    """
824    Determine whether current Python interpreter is running inside a virtual environment.
825    """
826    import sys
827    return (
828        hasattr(sys, 'real_prefix') or (
829            hasattr(sys, 'base_prefix')
830                and sys.base_prefix != sys.prefix
831        )
832    )

Determine whether current Python interpreter is running inside a virtual environment.

def is_venv_active(venv: str = 'mrsm', color: bool = True, debug: bool = False) -> bool:
215def is_venv_active(
216    venv: str = 'mrsm',
217    color : bool = True,
218    debug: bool = False
219) -> bool:
220    """
221    Check if a virtual environment is active.
222
223    Parameters
224    ----------
225    venv: str, default 'mrsm'
226        The virtual environment to check.
227
228    color: bool, default True
229        If `True`, include color in debug text.
230
231    debug: bool, default False
232        Verbosity toggle.
233
234    Returns
235    -------
236    A bool indicating whether the virtual environment `venv` is active.
237
238    """
239    return venv in active_venvs

Check if a virtual environment is active.

Parameters
  • venv (str, default 'mrsm'): The virtual environment to check.
  • color (bool, default True): If True, include color in debug text.
  • debug (bool, default False): Verbosity toggle.
Returns
  • A bool indicating whether the virtual environment venv is active.
def venv_exec( code: str, venv: Optional[str] = 'mrsm', env: Optional[Dict[str, str]] = None, with_extras: bool = False, as_proc: bool = False, capture_output: bool = True, debug: bool = False) -> Union[bool, Tuple[int, bytes, bytes], subprocess.Popen]:
614def venv_exec(
615    code: str,
616    venv: Optional[str] = 'mrsm',
617    env: Optional[Dict[str, str]] = None,
618    with_extras: bool = False,
619    as_proc: bool = False,
620    capture_output: bool = True,
621    debug: bool = False,
622) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
623    """
624    Execute Python code in a subprocess via a virtual environment's interpeter.
625    Return `True` if the code successfully executes, `False` on failure.
626
627    Parameters
628    ----------
629    code: str
630        The Python code to excecute.
631
632    venv: str, default 'mrsm'
633        The virtual environment to use to get the path for the Python executable.
634        If `venv` is `None`, use the default `sys.executable` path.
635
636    env: Optional[Dict[str, str]], default None
637        Optionally specify the environment variables for the subprocess.
638        Defaults to `os.environ`.
639
640    with_extras: bool, default False
641        If `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
642
643    as_proc: bool, default False
644        If `True`, return the `subprocess.Popen` object instead of executing.
645
646    Returns
647    -------
648    By default, return a bool indicating success.
649    If `as_proc` is `True`, return a `subprocess.Popen` object.
650    If `with_extras` is `True`, return a tuple of the exit code, stdout bytes, and stderr bytes.
651
652    """
653    import os
654    import subprocess
655    import platform
656    from meerschaum.utils.debug import dprint
657    from meerschaum.utils.process import _child_processes
658
659    executable = venv_executable(venv=venv)
660    cmd_list = [executable, '-c', code]
661    if env is None:
662        env = os.environ
663    if debug:
664        dprint(str(cmd_list))
665    if not with_extras and not as_proc:
666        return subprocess.call(cmd_list, env=env) == 0
667
668    stdout, stderr = (None, None) if not capture_output else (subprocess.PIPE, subprocess.PIPE)
669    group_kwargs = (
670        {
671            'preexec_fn': os.setsid,
672        } if platform.system() != 'Windows'
673        else {
674            'creationflags': CREATE_NEW_PROCESS_GROUP,
675        }
676    )
677    process = subprocess.Popen(
678        cmd_list,
679        stdout=stdout,
680        stderr=stderr,
681        stdin=sys.stdin,
682        env=env,
683        **group_kwargs
684    )
685    if as_proc:
686        _child_processes.append(process)
687        return process
688    stdout, stderr = process.communicate()
689    exit_code = process.returncode
690    return exit_code, stdout, stderr

Execute Python code in a subprocess via a virtual environment's interpeter. Return True if the code successfully executes, False on failure.

Parameters
  • code (str): The Python code to excecute.
  • venv (str, default 'mrsm'): The virtual environment to use to get the path for the Python executable. If venv is None, use the default sys.executable path.
  • env (Optional[Dict[str, str]], default None): Optionally specify the environment variables for the subprocess. Defaults to os.environ.
  • with_extras (bool, default False): If True, return a tuple of the exit code, stdout bytes, and stderr bytes.
  • as_proc (bool, default False): If True, return the subprocess.Popen object instead of executing.
Returns
  • By default, return a bool indicating success.
  • If as_proc is True, return a subprocess.Popen object.
  • If with_extras is True, return a tuple of the exit code, stdout bytes, and stderr bytes.
def venv_executable(venv: Optional[str] = 'mrsm') -> str:
589def venv_executable(venv: Optional[str] = 'mrsm') -> str:
590    """
591    The Python interpreter executable for a given virtual environment.
592    """
593    import sys
594    import platform
595    import meerschaum.config.paths as paths
596    return (
597        sys.executable if venv is None
598        else str(
599            paths.VIRTENV_RESOURCES_PATH
600            / venv
601            / (
602                'bin' if platform.system() != 'Windows'
603                else 'Scripts'
604            ) / (
605                'python'
606                + str(sys.version_info.major)
607                + '.'
608                + str(sys.version_info.minor)
609            )
610        )
611    )

The Python interpreter executable for a given virtual environment.

def venv_exists(venv: Optional[str], debug: bool = False) -> bool:
693def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
694    """
695    Determine whether a virtual environment has been created.
696    """
697    target_path = venv_target_path(venv, allow_nonexistent=True, debug=debug)
698    return target_path.exists()

Determine whether a virtual environment has been created.

def venv_target_path( venv: Optional[str], allow_nonexistent: bool = False, debug: bool = False) -> pathlib.Path:
701def venv_target_path(
702    venv: Union[str, None],
703    allow_nonexistent: bool = False,
704    debug: bool = False,
705) -> pathlib.Path:
706    """
707    Return a virtual environment's site-package path.
708
709    Parameters
710    ----------
711    venv: Union[str, None]
712        The virtual environment for which a path should be returned.
713
714    allow_nonexistent: bool, default False
715        If `True`, return a path even if it does not exist.
716
717    Returns
718    -------
719    The `pathlib.Path` object for the virtual environment's path.
720
721    """
722    import os
723    import sys
724    import platform
725    import site
726
727    import meerschaum.config.paths as paths
728    from meerschaum._internal.static import STATIC_CONFIG
729
730    ### Check sys.path for a user-writable site-packages directory.
731    if venv is None:
732
733        ### Return the known value for the portable environment.
734        environment_runtime = STATIC_CONFIG['environment']['runtime']
735        if os.environ.get(environment_runtime, None) == 'portable':
736            python_version_folder = (
737                'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
738            )
739            executable_path = pathlib.Path(sys.executable)
740            site_packages_path = (
741                (
742                    executable_path.parent.parent / 'lib' / python_version_folder / 'site-packages'
743                ) if platform.system() != 'Windows' else (
744                    executable_path.parent / 'Lib' / 'site-packages'
745                )
746            )
747            if not site_packages_path.exists():
748                raise EnvironmentError(f"Could not find '{site_packages_path}'. Does it exist?")
749            return site_packages_path
750
751        if not inside_venv():
752            user_site_packages = site.getusersitepackages()
753            if user_site_packages is None:
754                raise EnvironmentError("Could not determine user site packages.")
755
756            site_path = pathlib.Path(user_site_packages)
757            if not site_path.exists():
758
759                ### Windows does not have `os.geteuid()`.
760                if platform.system() == 'Windows' or os.geteuid() != 0:
761                    site_path.mkdir(parents=True, exist_ok=True)
762                    return site_path
763
764                ### Allow for dist-level paths (running as root).
765                for possible_dist in site.getsitepackages():
766                    dist_path = pathlib.Path(possible_dist)
767                    if not dist_path.exists():
768                        continue
769                    return dist_path
770                
771                raise EnvironmentError("Could not determine the dist-packages directory.")
772
773            return site_path
774
775    venv_root_path = (
776        (paths.VIRTENV_RESOURCES_PATH / venv)
777        if venv is not None
778        else pathlib.Path(sys.prefix)
779    )
780    target_path = venv_root_path
781
782    ### Ensure 'lib' or 'Lib' exists.
783    lib = 'lib' if platform.system() != 'Windows' else 'Lib'
784    if not allow_nonexistent:
785        if not venv_root_path.exists() or lib not in os.listdir(venv_root_path):
786            print(f"Failed to find lib directory for virtual environment '{venv}'.")
787            import traceback
788            traceback.print_stack()
789            sys.exit(1)
790    target_path = target_path / lib
791
792    ### Check if a 'python3.x' folder exists.
793    python_folder = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
794    if target_path.exists():
795        target_path = (
796            (target_path / python_folder)
797            if python_folder in os.listdir(target_path)
798            else target_path
799        )
800    else:
801        target_path = (
802            (target_path / python_folder)
803            if platform.system() != 'Windows'
804            else target_path
805        )
806
807    ### Ensure 'site-packages' exists.
808    if allow_nonexistent or 'site-packages' in os.listdir(target_path): ### Windows
809        target_path = target_path / 'site-packages'
810    else:
811        import traceback
812        traceback.print_stack()
813        print(f"Failed to find site-packages directory for virtual environment '{venv}'.")
814        print("This may be because you are using a different Python version.")
815        print("Try deleting the following directory and restarting Meerschaum:")
816        print(paths.VIRTENV_RESOURCES_PATH)
817        sys.exit(1)
818
819    return target_path

Return a virtual environment's site-package path.

Parameters
  • venv (Union[str, None]): The virtual environment for which a path should be returned.
  • allow_nonexistent (bool, default False): If True, return a path even if it does not exist.
Returns
  • The pathlib.Path object for the virtual environment's path.
def verify_venv(venv: str, debug: bool = False) -> None:
243def verify_venv(
244    venv: str,
245    debug: bool = False,
246) -> None:
247    """
248    Verify that the virtual environment matches the expected state.
249    """
250    import platform
251    import os
252    import shutil
253    import sys
254
255    import meerschaum.config.paths as paths
256    from meerschaum.utils.process import run_process
257    from meerschaum.utils.misc import make_symlink, is_symlink
258    from meerschaum.utils.warnings import warn
259
260    venv_path = paths.VIRTENV_RESOURCES_PATH / venv
261    bin_path = venv_path / (
262        'bin' if platform.system() != 'Windows' else "Scripts"
263    )
264    current_python_versioned_name = (
265        'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
266        + ('' if platform.system() != 'Windows' else '.exe')
267    )
268
269    if not (bin_path / current_python_versioned_name).exists():
270        init_venv(venv, verify=False, force=True, debug=debug)
271        current_python_in_venv_path = pathlib.Path(venv_executable(venv=venv))
272        current_python_in_sys_path = pathlib.Path(venv_executable(venv=None))
273        if not current_python_in_venv_path.exists():
274            if is_symlink(current_python_in_venv_path):
275                try:
276                    current_python_in_venv_path.unlink()
277                except Exception as e:
278                    print(f"Unable to remove symlink {current_python_in_venv_path}:\n{e}")
279            try:
280                make_symlink(current_python_in_sys_path, current_python_in_venv_path)
281            except Exception:
282                print(
283                    f"Unable to create symlink {current_python_in_venv_path} "
284                    + f"to {current_python_in_sys_path}."
285                )
286        files_to_inspect = sorted(os.listdir(bin_path), reverse=True)
287    else:
288        files_to_inspect = [current_python_versioned_name]
289
290    def get_python_version(python_path: pathlib.Path) -> Union[str, None]:
291        """
292        Return the version for the python binary at the given path.
293        """
294        try:
295            ### It might be a broken symlink, so skip on errors.
296            if debug:
297                print(f"Getting python version for {python_path}")
298            proc = run_process(
299                [str(python_path), '-V'],
300                as_proc=True,
301                capture_output=True,
302            )
303            stdout, stderr = proc.communicate(timeout=1.0)
304        except Exception as e:
305            ### E.g. the symlink may be broken.
306            if is_symlink(python_path):
307                try:
308                    python_path.unlink()
309                except Exception as _e:
310                    print(f"Unable to remove broken symlink {python_path}:\n{e}\n{_e}")
311            return None
312        return stdout.decode('utf-8').strip().replace('Python ', '')
313
314    ### Ensure the versions are symlinked correctly.
315    for filename in files_to_inspect:
316        if not filename.startswith('python'):
317            continue
318        python_path = bin_path / filename
319        version = get_python_version(python_path)
320        if version is None:
321            continue
322        try:
323            major_version = version.split('.', maxsplit=1)[0]
324            minor_version = version.split('.', maxsplit=2)[1]
325        except IndexError:
326            return
327        python_versioned_name = (
328            'python' + major_version + '.' + minor_version
329            + ('' if platform.system() != 'Windows' else '.exe')
330        )
331
332        ### E.g. python3.10 actually links to Python 3.10.
333        if filename == python_versioned_name:
334            try:
335                real_path = pathlib.Path(os.path.realpath(python_path))
336                real_path_exists = real_path.exists()
337            except Exception:
338                real_path_exists = False
339
340            if not real_path_exists:
341                try:
342                    python_path.unlink()
343                except Exception:
344                    pass
345                init_venv(venv, verify=False, force=True, debug=debug)
346                if not python_path.exists():
347                    raise FileNotFoundError(f"Unable to verify Python symlink:\n{python_path}")
348
349            if python_path == real_path:
350                continue
351
352            try:
353                python_path.unlink()
354            except Exception:
355                pass
356            success, msg = make_symlink(real_path, python_path)
357            if not success:
358                warn(msg, color=False)
359            continue
360
361        python_versioned_path = bin_path / python_versioned_name
362        if python_versioned_path.exists():
363            ### Avoid circular symlinks.
364            if get_python_version(python_versioned_path) == version:
365                continue
366            python_versioned_path.unlink()
367        shutil.move(python_path, python_versioned_path)

Verify that the virtual environment matches the expected state.