salt/tools/pkg/build.py

"""
These commands are used to build the salt onedir and system packages.
"""

# pylint: disable=resource-leakage,broad-except
from __future__ import annotations

import json
import logging
import os
import pathlib
import shutil
import tarfile
import zipfile
from typing import TYPE_CHECKING

from ptscripts import Context, command_group

import tools.utils

log = logging.getLogger(__name__)

# Define the command group
build = command_group(
    name="build",
    help="Package build related commands.",
    description=__doc__,
    parent="pkg",
)


@build.command(
    name="deb",
    arguments={
        "onedir": {
            "help": "The path to the onedir artifact",
        },
        "relenv_version": {
            "help": "The version of relenv to use",
        },
        "python_version": {
            "help": "The version of python to build with using relenv",
        },
        "arch": {
            "help": "The arch to build for",
        },
    },
)
def debian(
    ctx: Context,
    onedir: str = None,  # pylint: disable=bad-whitespace
    relenv_version: str = None,
    python_version: str = None,
    arch: str = None,
):
    """
    Build the deb package.
    """
    checkout = pathlib.Path.cwd()
    env_args = ["-e", "SALT_ONEDIR_ARCHIVE"]
    if onedir:
        onedir_artifact = checkout / "artifacts" / onedir
        _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)
        ctx.info(
            f"Building the package using the onedir artifact {str(onedir_artifact)}"
        )
        os.environ["SALT_ONEDIR_ARCHIVE"] = str(onedir_artifact)
    else:
        if arch is None:
            ctx.error(
                "Building the package from the source files but the arch to build for has not been given"
            )
            ctx.exit(1)
        ctx.info("Building the package from the source files")
        shared_constants = tools.utils.get_cicd_shared_context()
        if not python_version:
            python_version = shared_constants["python_version"]
        if not relenv_version:
            relenv_version = shared_constants["relenv_version"]
        if TYPE_CHECKING:
            assert python_version
            assert relenv_version
        new_env = {
            "SALT_RELENV_VERSION": relenv_version,
            "SALT_PYTHON_VERSION": python_version,
            "SALT_PACKAGE_ARCH": str(arch),
            "RELENV_FETCH_VERSION": relenv_version,
        }
        for key, value in new_env.items():
            os.environ[key] = value
            env_args.extend(["-e", key])

    env = os.environ.copy()
    env["PIP_CONSTRAINT"] = str(
        tools.utils.REPO_ROOT / "requirements" / "constraints.txt"
    )

    ctx.run("ln", "-sf", "pkg/debian/", ".")
    ctx.run("debuild", *env_args, "-uc", "-us", env=env)

    ctx.info("Done")


@build.command(
    name="rpm",
    arguments={
        "onedir": {
            "help": "The path to the onedir artifact",
        },
        "relenv_version": {
            "help": "The version of relenv to use",
        },
        "python_version": {
            "help": "The version of python to build with using relenv",
        },
        "arch": {
            "help": "The arch to build for",
        },
    },
)
def rpm(
    ctx: Context,
    onedir: str = None,  # pylint: disable=bad-whitespace
    relenv_version: str = None,
    python_version: str = None,
    arch: str = None,
):
    """
    Build the RPM package.
    """
    checkout = pathlib.Path.cwd()
    if onedir:
        onedir_artifact = checkout / "artifacts" / onedir
        _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)
        ctx.info(
            f"Building the package using the onedir artifact {str(onedir_artifact)}"
        )
        os.environ["SALT_ONEDIR_ARCHIVE"] = str(onedir_artifact)
    else:
        ctx.info("Building the package from the source files")
        if arch is None:
            ctx.error(
                "Building the package from the source files but the arch to build for has not been given"
            )
            ctx.exit(1)
        ctx.info("Building the package from the source files")
        shared_constants = tools.utils.get_cicd_shared_context()
        if not python_version:
            python_version = shared_constants["python_version"]
        if not relenv_version:
            relenv_version = shared_constants["relenv_version"]
        if TYPE_CHECKING:
            assert python_version
            assert relenv_version
        new_env = {
            "SALT_RELENV_VERSION": relenv_version,
            "SALT_PYTHON_VERSION": python_version,
            "SALT_PACKAGE_ARCH": str(arch),
            "RELENV_FETCH_VERSION": relenv_version,
        }
        for key, value in new_env.items():
            os.environ[key] = value

    env = os.environ.copy()
    env["PIP_CONSTRAINT"] = str(
        tools.utils.REPO_ROOT / "requirements" / "constraints.txt"
    )
    spec_file = checkout / "pkg" / "rpm" / "salt.spec"
    ctx.run(
        "rpmbuild", "-bb", f"--define=_salt_src {checkout}", str(spec_file), env=env
    )

    ctx.info("Done")


@build.command(
    name="macos",
    arguments={
        "onedir": {
            "help": "The name of the onedir artifact, if given it should be under artifacts/",
        },
        "salt_version": {
            "help": (
                "The salt version for which to build the repository configuration files. "
                "If not passed, it will be discovered by running 'python3 salt/version.py'."
            ),
            "required": True,
        },
        "sign": {
            "help": "Sign and notorize built package",
        },
        "relenv_version": {
            "help": "The version of relenv to use",
        },
        "python_version": {
            "help": "The version of python to build with using relenv",
        },
    },
)
def macos(
    ctx: Context,
    onedir: str = None,
    salt_version: str = None,
    sign: bool = False,
    relenv_version: str = None,
    python_version: str = None,
):
    """
    Build the macOS package.
    """
    if TYPE_CHECKING:
        assert onedir is not None
        assert salt_version is not None

    checkout = pathlib.Path.cwd()
    if onedir:
        onedir_artifact = checkout / "artifacts" / onedir
        ctx.info(f"Building package from existing onedir: {str(onedir_artifact)}")
        _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)

        build_root = checkout / "pkg" / "macos" / "build" / "opt"
        build_root.mkdir(parents=True, exist_ok=True)
        ctx.info(f"Extracting the onedir artifact to {build_root}")
        with tarfile.open(str(onedir_artifact)) as tarball:
            with ctx.chdir(onedir_artifact.parent):
                tarball.extractall(path=build_root)  # nosec
    else:
        ctx.info("Building package without an existing onedir")

    if not onedir:
        # Prep the salt onedir if not building from an existing one
        shared_constants = tools.utils.get_cicd_shared_context()
        if not python_version:
            python_version = shared_constants["python_version"]
        if not relenv_version:
            relenv_version = shared_constants["relenv_version"]
        if TYPE_CHECKING:
            assert python_version
            assert relenv_version
        os.environ["RELENV_FETCH_VERSION"] = relenv_version
        with ctx.chdir(checkout / "pkg" / "macos"):
            ctx.info("Fetching relenv python")
            ctx.run(
                "./build_python.sh",
                "--version",
                python_version,
                "--relenv-version",
                relenv_version,
            )

            ctx.info("Installing salt into the relenv python")
            ctx.run("./install_salt.sh")

    if sign:
        ctx.info("Signing binaries")
        with ctx.chdir(checkout / "pkg" / "macos"):
            ctx.run("./sign_binaries.sh")
    ctx.info("Building the macos package")
    with ctx.chdir(checkout / "pkg" / "macos"):
        ctx.run("./prep_salt.sh")
        if sign:
            package_args = ["--sign", salt_version]
        else:
            package_args = [salt_version]
        ctx.run("./package.sh", *package_args)
    if sign:
        ctx.info("Notarizing package")
        ret = ctx.run("uname", "-m", capture=True)
        cpu_arch = ret.stdout.strip().decode()
        with ctx.chdir(checkout / "pkg" / "macos"):
            ctx.run("./notarize.sh", f"salt-{salt_version}-py3-{cpu_arch}.pkg")

    ctx.info("Done")


@build.command(
    name="windows",
    arguments={
        "onedir": {
            "help": "The name of the onedir artifact, if given it should be under artifacts/",
        },
        "salt_version": {
            "help": (
                "The salt version for which to build the repository configuration files. "
                "If not passed, it will be discovered by running 'python3 salt/version.py'."
            ),
            "required": True,
        },
        "arch": {
            "help": "The architecture to build the package for",
            "choices": ("x86", "amd64"),
            "required": True,
        },
        "sign": {
            "help": "Sign and notarize built package",
        },
        "relenv_version": {
            "help": "The version of relenv to use",
        },
        "python_version": {
            "help": "The version of python to build with using relenv",
        },
    },
)
def windows(
    ctx: Context,
    onedir: str = None,
    salt_version: str = None,
    arch: str = None,
    sign: bool = False,
    relenv_version: str = None,
    python_version: str = None,
):
    """
    Build the Windows package.
    """
    if TYPE_CHECKING:
        assert salt_version is not None
        assert arch is not None

    shared_constants = tools.utils.get_cicd_shared_context()
    if not python_version:
        python_version = shared_constants["python_version"]
    if not relenv_version:
        relenv_version = shared_constants["relenv_version"]
    if TYPE_CHECKING:
        assert python_version
        assert relenv_version
    os.environ["RELENV_FETCH_VERSION"] = relenv_version

    build_cmd = [
        "powershell.exe",
        "&",
        "pkg/windows/build.cmd",
        "-Architecture",
        arch,
        "-Version",
        salt_version,
        "-PythonVersion",
        python_version,
        "-RelenvVersion",
        relenv_version,
        "-CICD",
    ]

    checkout = pathlib.Path.cwd()
    if onedir:
        build_cmd.append("-SkipInstall")
        onedir_artifact = checkout / "artifacts" / onedir
        ctx.info(f"Building package from existing onedir: {str(onedir_artifact)}")
        _check_pkg_build_files_exist(ctx, onedir_artifact=onedir_artifact)

        unzip_dir = checkout / "pkg" / "windows"
        ctx.info(f"Unzipping the onedir artifact to {unzip_dir}")
        with zipfile.ZipFile(onedir_artifact, mode="r") as archive:
            archive.extractall(unzip_dir)  # nosec

        move_dir = unzip_dir / "salt"
        build_env = unzip_dir / "buildenv"
        _check_pkg_build_files_exist(ctx, move_dir=move_dir)

        ctx.info(f"Moving {move_dir} directory to the build environment in {build_env}")
        shutil.move(move_dir, build_env)
    else:
        build_cmd.append("-Build")
        ctx.info("Building package without an existing onedir")

    ctx.info(f"Running: {' '.join(build_cmd)} ...")
    ctx.run(*build_cmd)

    if sign:
        env = os.environ.copy()
        envpath = env.get("PATH")
        if envpath is None:
            path_parts = []
        else:
            path_parts = envpath.split(os.pathsep)
        path_parts.extend(
            [
                r"C:\Program Files (x86)\Windows Kits\10\App Certification Kit",
                r"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools",
                r"C:\Program Files\DigiCert\DigiCert One Signing Manager Tools",
            ]
        )
        env["PATH"] = os.pathsep.join(path_parts)
        command = ["smksp_registrar.exe", "list"]
        ctx.info(f"Running: '{' '.join(command)}' ...")
        ctx.run(*command, env=env)
        command = ["smctl.exe", "keypair", "ls"]
        ctx.info(f"Running: '{' '.join(command)}' ...")
        ret = ctx.run(*command, env=env, check=False)
        if ret.returncode:
            ctx.error(f"Failed to run '{' '.join(command)}'")
        command = [
            r"C:\Windows\System32\certutil.exe",
            "-csp",
            "DigiCert Signing Manager KSP",
            "-key",
            "-user",
        ]
        ctx.info(f"Running: '{' '.join(command)}' ...")
        ret = ctx.run(*command, env=env, check=False)
        if ret.returncode:
            ctx.error(f"Failed to run '{' '.join(command)}'")

        command = ["smksp_cert_sync.exe"]
        ctx.info(f"Running: '{' '.join(command)}' ...")
        ret = ctx.run(*command, env=env, check=False)
        if ret.returncode:
            ctx.error(f"Failed to run '{' '.join(command)}'")

        for fname in (
            f"pkg/windows/build/Salt-Minion-{salt_version}-Py3-{arch}-Setup.exe",
            f"pkg/windows/build/Salt-Minion-{salt_version}-Py3-{arch}.msi",
        ):
            fpath = str(pathlib.Path(fname).resolve())
            ctx.info(f"Signing {fname} ...")
            ctx.run(
                "signtool.exe",
                "sign",
                "/sha1",
                os.environ["WIN_SIGN_CERT_SHA1_HASH"],
                "/tr",
                "http://timestamp.digicert.com",
                "/td",
                "SHA256",
                "/fd",
                "SHA256",
                fpath,
                env=env,
            )
            ctx.info(f"Verifying {fname} ...")
            ctx.run("signtool.exe", "verify", "/v", "/pa", fpath, env=env)

    ctx.info("Done")


@build.command(
    name="onedir-dependencies",
    arguments={
        "arch": {
            "help": "The architecture to build the package for",
            "choices": ("x86_64", "arm64", "x86", "amd64"),
            "required": True,
        },
        "python_version": {
            "help": "The version of python to create an environment for using relenv",
            "required": True,
        },
        "relenv_version": {
            "help": "The version of relenv to use",
        },
        "package_name": {
            "help": "The name of the relenv environment to be created",
            "required": True,
        },
        "platform": {
            "help": "The platform the relenv environment is being created on",
            "required": True,
        },
    },
)
def onedir_dependencies(
    ctx: Context,
    arch: str = None,
    python_version: str = None,
    relenv_version: str = None,
    package_name: str = None,
    platform: str = None,
):
    """
    Create a relenv environment with the onedir dependencies installed.

    NOTE: relenv needs to be installed into your environment and builds and toolchains (linux) fetched.
    """
    if TYPE_CHECKING:
        assert arch is not None
        assert python_version is not None
        assert package_name is not None
        assert platform is not None

    if platform == "darwin":
        platform = "macos"

    if platform != "macos" and arch == "arm64":
        arch = "aarch64"

    shared_constants = tools.utils.get_cicd_shared_context()
    if not python_version:
        python_version = shared_constants["python_version"]
    if not relenv_version:
        relenv_version = shared_constants["relenv_version"]
    if TYPE_CHECKING:
        assert python_version
        assert relenv_version
    os.environ["RELENV_FETCH_VERSION"] = relenv_version

    # We import relenv here because it is not a hard requirement for the rest of the tools commands
    try:
        import relenv.create
    except ImportError:
        ctx.exit(1, "Relenv not installed in the current environment.")

    dest = pathlib.Path(package_name).resolve()
    relenv.create.create(dest, arch=arch, version=python_version)

    # Validate that we're using the relenv version we really want to
    if platform == "windows":
        env_scripts_dir = dest / "Scripts"
    else:
        env_scripts_dir = dest / "bin"

    ret = ctx.run(
        str(env_scripts_dir / "relenv"), "--version", capture=True, check=False
    )
    if ret.returncode:
        ctx.error(f"Failed to get the relenv version: {ret}")
        ctx.exit(1)

    env_relenv_version = ret.stdout.strip().decode()
    if env_relenv_version != relenv_version:
        ctx.error(
            f"The onedir installed relenv version({env_relenv_version}) is not "
            f"the relenv version which should be used({relenv_version})."
        )
        ctx.exit(1)

    ctx.info(
        f"The relenv version installed in the onedir env({env_relenv_version}) "
        f"matches the version which must be used."
    )

    env = os.environ.copy()
    install_args = ["-v"]
    if platform == "windows":
        python_bin = env_scripts_dir / "python"
    else:
        env["RELENV_BUILDENV"] = "1"
        python_bin = env_scripts_dir / "python3"
        install_args.extend(
            [
                "--use-pep517",
                "--no-cache-dir",
                "--no-binary=:all:",
            ]
        )

    version_info = ctx.run(
        str(python_bin),
        "-c",
        "import sys; print('{}.{}'.format(*sys.version_info))",
        capture=True,
    )
    requirements_version = version_info.stdout.strip().decode()
    requirements_file = (
        tools.utils.REPO_ROOT
        / "requirements"
        / "static"
        / "pkg"
        / f"py{requirements_version}"
        / f"{platform if platform != 'macos' else 'darwin'}.txt"
    )
    _check_pkg_build_files_exist(ctx, requirements_file=requirements_file)

    env["PIP_CONSTRAINT"] = str(
        tools.utils.REPO_ROOT / "requirements" / "constraints.txt"
    )
    ctx.run(
        str(python_bin),
        "-m",
        "pip",
        "install",
        "-U",
        "setuptools",
        "pip",
        "wheel",
        env=env,
    )
    ctx.run(
        str(python_bin),
        "-m",
        "pip",
        "install",
        *install_args,
        "-r",
        str(requirements_file),
        env=env,
    )


@build.command(
    name="salt-onedir",
    arguments={
        "salt_name": {
            "help": "The path to the salt code to install, relative to the repo root",
        },
        "platform": {
            "help": "The platform that installed is being installed on",
            "required": True,
        },
        "package_name": {
            "help": "The name of the relenv environment to install salt into",
            "required": True,
        },
        "relenv_version": {
            "help": "The version of relenv to use",
        },
    },
)
def salt_onedir(
    ctx: Context,
    salt_name: str,
    platform: str = None,
    package_name: str = None,
    relenv_version: str = None,
):
    """
    Install salt into a relenv onedir environment.
    """
    if TYPE_CHECKING:
        assert platform is not None
        assert package_name is not None

    if platform == "darwin":
        platform = "macos"

    shared_constants = tools.utils.get_cicd_shared_context()
    if not relenv_version:
        relenv_version = shared_constants["relenv_version"]
    if TYPE_CHECKING:
        assert relenv_version
    os.environ["RELENV_FETCH_VERSION"] = relenv_version

    salt_archive = pathlib.Path(salt_name).resolve()
    onedir_env = pathlib.Path(package_name).resolve()
    _check_pkg_build_files_exist(ctx, onedir_env=onedir_env, salt_archive=salt_archive)

    # Validate that we're using the relenv version we really want to
    if platform == "windows":
        env_scripts_dir = onedir_env / "Scripts"
    else:
        env_scripts_dir = onedir_env / "bin"

    ret = ctx.run(
        str(env_scripts_dir / "relenv"), "--version", capture=True, check=False
    )
    if ret.returncode:
        ctx.error(f"Failed to get the relenv version: {ret}")
        ctx.exit(1)

    env_relenv_version = ret.stdout.strip().decode()
    if env_relenv_version != relenv_version:
        ctx.error(
            f"The onedir installed relenv version({env_relenv_version}) is not "
            f"the relenv version which should be used({relenv_version})."
        )
        ctx.exit(1)

    ctx.info(
        f"The relenv version installed in the onedir env({env_relenv_version}) "
        f"matches the version which must be used."
    )

    env = os.environ.copy()
    env["USE_STATIC_REQUIREMENTS"] = "1"
    env["RELENV_BUILDENV"] = "1"
    if platform == "windows":
        ctx.run(
            "powershell.exe",
            r"pkg\windows\install_salt.cmd",
            "-BuildDir",
            str(onedir_env),
            "-CICD",
            "-SourceTarball",
            str(salt_archive),
            env=env,
        )
        ctx.run(
            "powershell.exe",
            r"pkg\windows\prep_salt.cmd",
            "-BuildDir",
            str(onedir_env),
            "-CICD",
            env=env,
        )
        python_executable = str(env_scripts_dir / "python.exe")
        ret = ctx.run(
            python_executable,
            "-c",
            "import json, sys, site, pathlib; sys.stdout.write(json.dumps([pathlib.Path(p).as_posix() for p in site.getsitepackages()]))",
            capture=True,
        )
        if ret.returncode:
            ctx.error(f"Failed to get the path to `site-packages`: {ret}")
            ctx.exit(1)
        site_packages_json = json.loads(ret.stdout.strip().decode())
        ctx.info(f"Discovered 'site-packages' paths: {site_packages_json}")
    else:
        env["RELENV_PIP_DIR"] = "1"
        pip_bin = env_scripts_dir / "pip3"
        ctx.run(
            str(pip_bin),
            "install",
            "--no-warn-script-location",
            str(salt_archive),
            env=env,
        )
        if platform == "macos":

            def errfn(fn, path, err):
                ctx.info(f"Removing {path} failed: {err}")

            for subdir in ("opt", "etc", "Library"):
                path = onedir_env / subdir
                if path.exists():
                    shutil.rmtree(path, onerror=errfn)

        python_executable = str(env_scripts_dir / "python3")
        ret = ctx.run(
            python_executable,
            "-c",
            "import json, sys, site, pathlib; sys.stdout.write(json.dumps(site.getsitepackages()))",
            capture=True,
        )
        if ret.returncode:
            ctx.error(f"Failed to get the path to `site-packages`: {ret}")
            ctx.exit(1)
        site_packages_json = json.loads(ret.stdout.strip().decode())
        ctx.info(f"Discovered 'site-packages' paths: {site_packages_json}")

    site_packages: str
    for site_packages_path in site_packages_json:
        if "site-packages" in site_packages_path:
            site_packages = site_packages_path
            break
    else:
        ctx.error("Cloud not find a site-packages path with 'site-packages' in it?!")
        ctx.exit(1)

    ret = ctx.run(
        str(python_executable),
        "-c",
        "import sys; print('{}.{}'.format(*sys.version_info))",
        capture=True,
    )
    python_version_info = ret.stdout.strip().decode()
    extras_dir = onedir_env / f"extras-{python_version_info}"
    ctx.info(f"Creating Salt's extras path: {extras_dir}")
    extras_dir.mkdir(exist_ok=True)

    for fname in ("_salt_onedir_extras.py", "_salt_onedir_extras.pth"):
        src = tools.utils.REPO_ROOT / "pkg" / "common" / "onedir" / fname
        dst = pathlib.Path(site_packages) / fname
        ctx.info(f"Copying '{src.relative_to(tools.utils.REPO_ROOT)}' to '{dst}' ...")
        shutil.copyfile(src, dst)

    # Add package type file for package grain
    with open(
        pathlib.Path(site_packages) / "salt" / "_pkg.txt", "w", encoding="utf-8"
    ) as fp:
        fp.write("onedir")


def _check_pkg_build_files_exist(ctx: Context, **kwargs):
    for name, path in kwargs.items():
        if not path.exists():
            ctx.error(f"The path {path} does not exist, {name} is not valid... exiting")
            ctx.exit(1)