mirror of
https://github.com/saltstack/salt.git
synced 2025-04-16 09:40:20 +00:00
Remove the custom pyinstaller support code needed for Salt < 3006.x
Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio
Pedro Algarvio
parent
5463132e9d
commit
43e17e62ac
11 changed files with 0 additions and 545 deletions
|
|
@ -1,21 +0,0 @@
|
|||
"""
|
||||
This module exists to help PyInstaller bundle Salt
|
||||
"""
|
||||
import pathlib
|
||||
|
||||
PYINSTALLER_UTILS_DIR_PATH = pathlib.Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def get_hook_dirs():
|
||||
"""
|
||||
Return a list of paths that PyInstaller can search for hooks.
|
||||
"""
|
||||
hook_dirs = {PYINSTALLER_UTILS_DIR_PATH}
|
||||
for path in PYINSTALLER_UTILS_DIR_PATH.iterdir():
|
||||
if not path.is_dir():
|
||||
continue
|
||||
if "__pycache__" in path.parts:
|
||||
continue
|
||||
hook_dirs.add(path)
|
||||
|
||||
return sorted(str(p) for p in hook_dirs)
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
# pylint: disable=3rd-party-module-not-gated
|
||||
|
||||
import logging
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from PyInstaller.utils import hooks
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _filter_stdlib_tests(name):
|
||||
"""
|
||||
Filter out non useful modules from the stdlib
|
||||
"""
|
||||
if ".test." in name:
|
||||
return False
|
||||
if ".tests." in name:
|
||||
return False
|
||||
if ".idle_test" in name:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _python_stdlib_path():
|
||||
"""
|
||||
Return the path to the standard library folder
|
||||
"""
|
||||
base_exec_prefix = pathlib.Path(sys.base_exec_prefix)
|
||||
log.info("Grabbing 'base_exec_prefix' for platform: %s", sys.platform)
|
||||
if not sys.platform.lower().startswith("win"):
|
||||
return base_exec_prefix / "lib" / "python{}.{}".format(*sys.version_info)
|
||||
return base_exec_prefix / "Lib"
|
||||
|
||||
|
||||
def _collect_python_stdlib_hidden_imports():
|
||||
"""
|
||||
Collect all of the standard library(most of it) as hidden imports.
|
||||
"""
|
||||
_hidden_imports = set()
|
||||
|
||||
stdlib = _python_stdlib_path()
|
||||
if not stdlib.exists():
|
||||
log.error("The path '%s' does not exist", stdlib)
|
||||
return list(_hidden_imports)
|
||||
|
||||
log.info(
|
||||
"Collecting hidden imports from the python standard library at: %s",
|
||||
stdlib,
|
||||
)
|
||||
for path in stdlib.glob("*"):
|
||||
if path.is_dir():
|
||||
if path.name in (
|
||||
"__pycache__",
|
||||
"site-packages",
|
||||
"test",
|
||||
"turtledemo",
|
||||
"ensurepip",
|
||||
):
|
||||
continue
|
||||
if path.joinpath("__init__.py").is_file():
|
||||
log.info("Collecting: %s", path.name)
|
||||
try:
|
||||
_module_hidden_imports = hooks.collect_submodules(
|
||||
path.name, filter=_filter_stdlib_tests
|
||||
)
|
||||
log.debug("Collected(%s): %s", path.name, _module_hidden_imports)
|
||||
_hidden_imports.update(set(_module_hidden_imports))
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
log.error("Failed to collect %r: %s", path.name, exc)
|
||||
continue
|
||||
if path.suffix not in (".py", ".pyc", ".pyo"):
|
||||
continue
|
||||
_hidden_imports.add(path.stem)
|
||||
log.info("Collected stdlib hidden imports: %s", sorted(_hidden_imports))
|
||||
return sorted(_hidden_imports)
|
||||
|
||||
|
||||
def _collect_python_stdlib_dynamic_libraries():
|
||||
"""
|
||||
Collect all of the standard library(most of it) dynamic libraries.
|
||||
"""
|
||||
_dynamic_libs = set()
|
||||
|
||||
stdlib = _python_stdlib_path()
|
||||
if not stdlib.exists():
|
||||
log.error("The path '%s' does not exist", stdlib)
|
||||
return list(_dynamic_libs)
|
||||
|
||||
log.info(
|
||||
"Collecting dynamic libraries from the python standard library at: %s",
|
||||
stdlib,
|
||||
)
|
||||
for path in stdlib.glob("*"):
|
||||
if not path.is_dir():
|
||||
continue
|
||||
if path.name in (
|
||||
"__pycache__",
|
||||
"site-packages",
|
||||
"test",
|
||||
"turtledemo",
|
||||
"ensurepip",
|
||||
):
|
||||
continue
|
||||
if path.joinpath("__init__.py").is_file():
|
||||
log.info("Collecting: %s", path.name)
|
||||
try:
|
||||
_module_dynamic_libs = hooks.collect_dynamic_libs(path.name, path.name)
|
||||
log.debug("Collected(%s): %s", path.name, _module_dynamic_libs)
|
||||
_dynamic_libs.update(set(_module_dynamic_libs))
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
log.error("Failed to collect %r: %s", path.name, exc)
|
||||
log.info("Collected stdlib dynamic libs: %s", sorted(_dynamic_libs))
|
||||
return sorted(_dynamic_libs)
|
||||
|
||||
|
||||
def _filter_submodules(name):
|
||||
# this should never happen, but serves as a place-holder for when/if we have to filter
|
||||
if not name.startswith("salt"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# Collect Salt datas, binaries(should be None) and hidden imports
|
||||
SALT_DATAS, SALT_BINARIES, SALT_HIDDENIMPORTS = hooks.collect_all(
|
||||
"salt",
|
||||
include_py_files=True,
|
||||
filter_submodules=_filter_submodules,
|
||||
)
|
||||
|
||||
# In case there's salt-extensions installed, collect their datas and hidden imports
|
||||
SALT_EXTENSIONS_DATAS, SALT_EXTENSIONS_HIDDENIMPORTS = hooks.collect_entry_point(
|
||||
"salt.loader"
|
||||
)
|
||||
|
||||
|
||||
# PyInstaller attributes
|
||||
datas = sorted(set(SALT_DATAS + SALT_EXTENSIONS_DATAS))
|
||||
binaries = sorted(set(SALT_BINARIES))
|
||||
hiddenimports = sorted(
|
||||
set(
|
||||
SALT_HIDDENIMPORTS
|
||||
+ SALT_EXTENSIONS_HIDDENIMPORTS
|
||||
+ _collect_python_stdlib_hidden_imports()
|
||||
)
|
||||
)
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
{
|
||||
"subprocess": ["pyi_rth_subprocess.py"],
|
||||
"salt.utils.vt": ["pyi_rth_salt.utils.vt.py"],
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
"""
|
||||
This package contains support code to package Salt with PyInstaller.
|
||||
"""
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
"""
|
||||
This package contains the runtime hooks support code for when Salt is pacakged with PyInstaller.
|
||||
"""
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import salt.utils.vt
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def clean_pyinstaller_vars(environ):
|
||||
"""
|
||||
Restore or cleanup PyInstaller specific environent variable behavior.
|
||||
"""
|
||||
if environ is None:
|
||||
environ = dict(os.environ)
|
||||
# When Salt is bundled with tiamat, it MUST NOT contain LD_LIBRARY_PATH
|
||||
# when shelling out, or, at least the value of LD_LIBRARY_PATH set by
|
||||
# pyinstaller.
|
||||
# See:
|
||||
# https://pyinstaller.readthedocs.io/en/stable/runtime-information.html#ld-library-path-libpath-considerations
|
||||
for varname in ("LD_LIBRARY_PATH", "LIBPATH"):
|
||||
original_varname = "{}_ORIG".format(varname)
|
||||
if varname in environ and environ[varname] == sys._MEIPASS:
|
||||
# If we find the varname on the user provided environment we need to at least
|
||||
# check if it's not the value set by PyInstaller, if it is, remove it.
|
||||
log.debug(
|
||||
"User provided environment variable %r with value %r which is "
|
||||
"the value that PyInstaller set's. Removing it",
|
||||
varname,
|
||||
environ[varname],
|
||||
)
|
||||
environ.pop(varname)
|
||||
|
||||
if original_varname in environ and varname not in environ:
|
||||
# We found the original variable set by PyInstaller, and we didn't find
|
||||
# any user provided variable, let's rename it.
|
||||
log.debug(
|
||||
"The %r variable was found in the passed environment, renaming it to %r",
|
||||
original_varname,
|
||||
varname,
|
||||
)
|
||||
environ[varname] = environ.pop(original_varname)
|
||||
|
||||
if varname not in environ:
|
||||
if original_varname in os.environ:
|
||||
log.debug(
|
||||
"Renaming environment variable %r to %r", original_varname, varname
|
||||
)
|
||||
environ[varname] = os.environ[original_varname]
|
||||
elif varname in os.environ:
|
||||
# Override the system environ variable with an empty one
|
||||
log.debug("Setting environment variable %r to an empty string", varname)
|
||||
environ[varname] = ""
|
||||
return environ
|
||||
|
||||
|
||||
class PyinstallerPopen(subprocess.Popen):
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs["env"] = clean_pyinstaller_vars(kwargs.pop("env", None))
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# From https://github.com/pyinstaller/pyinstaller/blob/v5.1/PyInstaller/hooks/rthooks/pyi_rth_subprocess.py
|
||||
#
|
||||
# In windowed mode, force any unused pipes (stdin, stdout and stderr) to be DEVNULL instead of inheriting the
|
||||
# invalid corresponding handles from this parent process.
|
||||
if sys.platform == "win32" and not isinstance(sys.stdout, io.IOBase):
|
||||
|
||||
def _get_handles(self, stdin, stdout, stderr):
|
||||
stdin, stdout, stderr = (
|
||||
subprocess.DEVNULL if pipe is None else pipe
|
||||
for pipe in (stdin, stdout, stderr)
|
||||
)
|
||||
return super()._get_handles(stdin, stdout, stderr)
|
||||
|
||||
|
||||
class PyinstallerTerminal(salt.utils.vt.Terminal): # pylint: disable=abstract-method
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs["env"] = clean_pyinstaller_vars(kwargs.pop("env", None))
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
"""
|
||||
PyInstaller runtime hook to patch salt.utils.vt.Terminal
|
||||
"""
|
||||
import logging
|
||||
|
||||
import salt.utils.vt
|
||||
from salt.utils.pyinstaller.rthooks._overrides import PyinstallerTerminal
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
# Patch salt.utils.vt.Terminal when running within a pyinstalled bundled package
|
||||
salt.utils.vt.Terminal = PyinstallerTerminal
|
||||
|
||||
log.debug("Replaced 'salt.utils.vt.Terminal' with 'PyinstallerTerminal'")
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
"""
|
||||
PyInstaller runtime hook to patch subprocess.Popen
|
||||
"""
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
from salt.utils.pyinstaller.rthooks._overrides import PyinstallerPopen
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
# Patch subprocess.Popen when running within a pyinstalled bundled package
|
||||
subprocess.Popen = PyinstallerPopen
|
||||
|
||||
log.debug("Replaced 'subprocess.Popen' with 'PyinstallerTerminal'")
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import salt.utils.pyinstaller.rthooks._overrides as overrides
|
||||
from tests.support import mock
|
||||
from tests.support.helpers import PatchedEnviron
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skip(reason="PyInstaller is no longer used."),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(params=("LD_LIBRARY_PATH", "LIBPATH"))
|
||||
def envvar(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def meipass(envvar):
|
||||
with mock.patch("salt.utils.pyinstaller.rthooks._overrides.sys") as patched_sys:
|
||||
patched_sys._MEIPASS = "{}_VALUE".format(envvar)
|
||||
assert overrides.sys._MEIPASS == "{}_VALUE".format(envvar)
|
||||
yield "{}_VALUE".format(envvar)
|
||||
assert not hasattr(sys, "_MEIPASS")
|
||||
assert not hasattr(overrides.sys, "_MEIPASS")
|
||||
|
||||
|
||||
def test_vt_terminal_environ_cleanup_original(envvar, meipass):
|
||||
orig_envvar = "{}_ORIG".format(envvar)
|
||||
with PatchedEnviron(**{orig_envvar: meipass}):
|
||||
original_env = dict(os.environ)
|
||||
assert orig_envvar in original_env
|
||||
instance = overrides.PyinstallerTerminal(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import os, json; print(json.dumps(dict(os.environ)))",
|
||||
],
|
||||
stream_stdout=False,
|
||||
stream_stderr=False,
|
||||
)
|
||||
buffer_o = buffer_e = ""
|
||||
while instance.has_unread_data:
|
||||
stdout, stderr = instance.recv()
|
||||
if stdout:
|
||||
buffer_o += stdout
|
||||
if stderr:
|
||||
buffer_e += stderr
|
||||
instance.terminate()
|
||||
|
||||
assert instance.exitstatus == 0
|
||||
returned_env = json.loads(buffer_o)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert orig_envvar not in returned_env
|
||||
assert returned_env[envvar] == meipass
|
||||
|
||||
|
||||
def test_vt_terminal_environ_cleanup_original_passed_directly(envvar, meipass):
|
||||
orig_envvar = "{}_ORIG".format(envvar)
|
||||
env = {
|
||||
orig_envvar: meipass,
|
||||
}
|
||||
original_env = dict(os.environ)
|
||||
|
||||
instance = overrides.PyinstallerTerminal(
|
||||
[sys.executable, "-c", "import os, json; print(json.dumps(dict(os.environ)))"],
|
||||
env=env.copy(),
|
||||
stream_stdout=False,
|
||||
stream_stderr=False,
|
||||
)
|
||||
buffer_o = buffer_e = ""
|
||||
while instance.has_unread_data:
|
||||
stdout, stderr = instance.recv()
|
||||
if stdout:
|
||||
buffer_o += stdout
|
||||
if stderr:
|
||||
buffer_e += stderr
|
||||
instance.terminate()
|
||||
|
||||
assert instance.exitstatus == 0
|
||||
returned_env = json.loads(buffer_o)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert orig_envvar not in returned_env
|
||||
assert returned_env[envvar] == meipass
|
||||
|
||||
|
||||
def test_vt_terminal_environ_cleanup(envvar, meipass):
|
||||
with PatchedEnviron(**{envvar: meipass}):
|
||||
original_env = dict(os.environ)
|
||||
assert envvar in original_env
|
||||
instance = overrides.PyinstallerTerminal(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import os, json; print(json.dumps(dict(os.environ)))",
|
||||
],
|
||||
stream_stdout=False,
|
||||
stream_stderr=False,
|
||||
)
|
||||
buffer_o = buffer_e = ""
|
||||
while instance.has_unread_data:
|
||||
stdout, stderr = instance.recv()
|
||||
if stdout:
|
||||
buffer_o += stdout
|
||||
if stderr:
|
||||
buffer_e += stderr
|
||||
instance.terminate()
|
||||
|
||||
assert instance.exitstatus == 0
|
||||
returned_env = json.loads(buffer_o)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert returned_env[envvar] == ""
|
||||
|
||||
|
||||
def test_vt_terminal_environ_cleanup_passed_directly_not_removed(envvar, meipass):
|
||||
env = {
|
||||
envvar: envvar,
|
||||
}
|
||||
original_env = dict(os.environ)
|
||||
|
||||
instance = overrides.PyinstallerTerminal(
|
||||
[sys.executable, "-c", "import os, json; print(json.dumps(dict(os.environ)))"],
|
||||
env=env.copy(),
|
||||
stream_stdout=False,
|
||||
stream_stderr=False,
|
||||
)
|
||||
buffer_o = buffer_e = ""
|
||||
while instance.has_unread_data:
|
||||
stdout, stderr = instance.recv()
|
||||
if stdout:
|
||||
buffer_o += stdout
|
||||
if stderr:
|
||||
buffer_e += stderr
|
||||
instance.terminate()
|
||||
|
||||
assert instance.exitstatus == 0
|
||||
returned_env = json.loads(buffer_o)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert returned_env[envvar] == envvar
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import salt.utils.pyinstaller.rthooks._overrides as overrides
|
||||
from tests.support import mock
|
||||
from tests.support.helpers import PatchedEnviron
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skip(reason="PyInstaller is no longer used."),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(params=("LD_LIBRARY_PATH", "LIBPATH"))
|
||||
def envvar(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def meipass(envvar):
|
||||
with mock.patch("salt.utils.pyinstaller.rthooks._overrides.sys") as patched_sys:
|
||||
patched_sys._MEIPASS = "{}_VALUE".format(envvar)
|
||||
assert overrides.sys._MEIPASS == "{}_VALUE".format(envvar)
|
||||
yield "{}_VALUE".format(envvar)
|
||||
assert not hasattr(sys, "_MEIPASS")
|
||||
assert not hasattr(overrides.sys, "_MEIPASS")
|
||||
|
||||
|
||||
def test_subprocess_popen_environ_cleanup_original(envvar, meipass):
|
||||
orig_envvar = "{}_ORIG".format(envvar)
|
||||
with PatchedEnviron(**{orig_envvar: meipass}):
|
||||
original_env = dict(os.environ)
|
||||
assert orig_envvar in original_env
|
||||
instance = overrides.PyinstallerPopen(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import os, json; print(json.dumps(dict(os.environ)))",
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
)
|
||||
stdout, _ = instance.communicate()
|
||||
assert instance.returncode == 0
|
||||
returned_env = json.loads(stdout)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert orig_envvar not in returned_env
|
||||
assert returned_env[envvar] == meipass
|
||||
|
||||
|
||||
def test_subprocess_popen_environ_cleanup_original_passed_directly(envvar, meipass):
|
||||
orig_envvar = "{}_ORIG".format(envvar)
|
||||
env = {
|
||||
orig_envvar: meipass,
|
||||
}
|
||||
original_env = dict(os.environ)
|
||||
|
||||
instance = overrides.PyinstallerPopen(
|
||||
[sys.executable, "-c", "import os, json; print(json.dumps(dict(os.environ)))"],
|
||||
env=env.copy(),
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
)
|
||||
stdout, _ = instance.communicate()
|
||||
assert instance.returncode == 0
|
||||
returned_env = json.loads(stdout)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert orig_envvar not in returned_env
|
||||
assert returned_env[envvar] == meipass
|
||||
|
||||
|
||||
def test_subprocess_popen_environ_cleanup(envvar, meipass):
|
||||
with PatchedEnviron(**{envvar: meipass}):
|
||||
original_env = dict(os.environ)
|
||||
assert envvar in original_env
|
||||
instance = overrides.PyinstallerPopen(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import os, json; print(json.dumps(dict(os.environ)))",
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
)
|
||||
stdout, _ = instance.communicate()
|
||||
assert instance.returncode == 0
|
||||
returned_env = json.loads(stdout)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert returned_env[envvar] == ""
|
||||
|
||||
|
||||
def test_subprocess_popen_environ_cleanup_passed_directly_not_removed(envvar, meipass):
|
||||
env = {
|
||||
envvar: envvar,
|
||||
}
|
||||
original_env = dict(os.environ)
|
||||
|
||||
instance = overrides.PyinstallerPopen(
|
||||
[sys.executable, "-c", "import os, json; print(json.dumps(dict(os.environ)))"],
|
||||
env=env.copy(),
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
)
|
||||
stdout, _ = instance.communicate()
|
||||
assert instance.returncode == 0
|
||||
returned_env = json.loads(stdout)
|
||||
assert returned_env != original_env
|
||||
assert envvar in returned_env
|
||||
assert returned_env[envvar] == envvar
|
||||
Loading…
Add table
Reference in a new issue